mrcharles
422999bf3c
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
1139 lines
50 KiB
Bash
Executable File
1139 lines
50 KiB
Bash
Executable File
#!/bin/bash
|
|
###############################################################################
|
|
# check.sh
|
|
#
|
|
# READ-ONLY, comprehensive Proxmox host assessment.
|
|
# - Installs NOTHING. Modifies NOTHING. Writes only its own log.
|
|
# - Detects what is/is isn't installed so the next step can be informed.
|
|
# - Runtime ~30-60 seconds on a busy node.
|
|
#
|
|
# Output: /root/<short-hostname>.log
|
|
#
|
|
# Usage (as root):
|
|
# bash check.sh
|
|
###############################################################################
|
|
set -u
|
|
umask 077
|
|
|
|
HOST="$(hostname -s)"
|
|
OUT="/root/${HOST}.log"
|
|
: > "$OUT"
|
|
TS="$(date -u +'%Y-%m-%dT%H:%M:%SZ')"
|
|
START_EPOCH="$(date +%s)"
|
|
|
|
# ===========================================================================
|
|
# Helpers (pattern from project memory)
|
|
# ===========================================================================
|
|
blank(){ printf '\n' >> "$OUT"; }
|
|
|
|
hdr() {
|
|
{
|
|
printf '\n'
|
|
printf '###############################################################################\n'
|
|
printf '# %s\n' "$1"
|
|
printf '###############################################################################\n'
|
|
} >> "$OUT"
|
|
}
|
|
|
|
# cmd LABEL CMD [ARGS...] - simple commands only (no pipes/loops/&&)
|
|
cmd() {
|
|
local label="$1"; shift
|
|
{
|
|
printf '\n--- %s\n' "$label"
|
|
printf ' $ %s\n' "$*"
|
|
} >> "$OUT"
|
|
"$@" >> "$OUT" 2>&1 || printf ' [exit=%s / not installed or not permitted]\n' "$?" >> "$OUT"
|
|
}
|
|
|
|
# cmdsh LABEL <<'EOF' ... EOF - arbitrary shell snippet (pipes, loops OK).
|
|
# Heredoc body MUST use quoted 'EOF' so $vars pass through to eval literally;
|
|
# export any outer-scope vars the body needs before the call.
|
|
# shellcheck disable=SC2310
|
|
cmdsh() {
|
|
local label="$1" snippet
|
|
snippet=$(cat)
|
|
{
|
|
printf '\n--- %s\n' "$label"
|
|
printf ' $ %s\n' "$snippet"
|
|
} >> "$OUT"
|
|
eval "$snippet" >> "$OUT" 2>&1 || printf ' [exit=%s]\n' "$?" >> "$OUT"
|
|
}
|
|
|
|
# filecat LABEL PATH [PATH...] - dump file contents (cap 300 lines each)
|
|
filecat() {
|
|
local label="$1"; shift
|
|
printf '\n--- %s\n' "$label" >> "$OUT"
|
|
local f
|
|
for f in "$@"; do
|
|
if [ -r "$f" ]; then
|
|
printf ' --- %s ---\n' "$f" >> "$OUT"
|
|
sed -n '1,300p' "$f" >> "$OUT"
|
|
else
|
|
printf ' [missing: %s]\n' "$f" >> "$OUT"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# dirlist LABEL PATH - list directory contents (existence + sizes)
|
|
dirlist() {
|
|
local label="$1" d="$2"
|
|
printf '\n--- %s\n' "$label" >> "$OUT"
|
|
if [ -d "$d" ]; then
|
|
ls -la "$d" >> "$OUT" 2>&1
|
|
else
|
|
printf ' [missing: %s]\n' "$d" >> "$OUT"
|
|
fi
|
|
}
|
|
|
|
# ===========================================================================
|
|
# Header
|
|
# ===========================================================================
|
|
{
|
|
printf '###############################################################################\n'
|
|
printf '# Proxmox VE host performance assessment (READ-ONLY)\n'
|
|
printf '# Host: %s (%s)\n' "$HOST" "$(hostname -f 2>/dev/null)"
|
|
printf '# FQDN: %s\n' "$(hostname -f 2>/dev/null)"
|
|
printf '# All IPs: %s\n' "$(hostname -I 2>/dev/null)"
|
|
printf '# Collected: %s (UTC)\n' "$TS"
|
|
printf '# Kernel: %s\n' "$(uname -r)"
|
|
printf '# Machine: %s %s\n' "$(uname -m)" "$(uname -o)"
|
|
printf '# Caller: %s (uid=%s)\n' "$(whoami)" "$(id -u)"
|
|
printf '# Script: check.sh - strictly READ-ONLY\n'
|
|
printf '###############################################################################\n'
|
|
} >> "$OUT"
|
|
|
|
# ===========================================================================
|
|
# 1. TOOL AVAILABILITY MATRIX
|
|
# (most important section: tells us what is/isn't installed)
|
|
# ===========================================================================
|
|
hdr "1. TOOL AVAILABILITY MATRIX"
|
|
|
|
cmdsh "command presence + path + version" <<'EOF'
|
|
# Each line: name | present? | path | version
|
|
printf '%-22s %-8s %-30s %s\n' "TOOL" "PRESENT" "PATH" "VERSION/NOTE"
|
|
printf '%-22s %-8s %-30s %s\n' "----" "-------" "----" "------------"
|
|
for t in \
|
|
pveversion pvecm qm pct pvesm pvesh pveperf proxmox-backup-client \
|
|
ip ss ethtool bridge brctl nmcli \
|
|
tcpdump mtr nstat netstat nicstat ifstat \
|
|
dmidecode lscpu lshw hwinfo inxi lspci lsusb numactl numastat \
|
|
cpupower turbostat x86_energy_perf_policy rdmsr \
|
|
tuned-adm irqbalance \
|
|
zpool zfs zdb arcstat arc_summary \
|
|
lsblk blkid blockdev findmnt smartctl nvme hdparm \
|
|
storcli storcli64 MegaCli MegaCli64 tw-cli arcconf \
|
|
sas2ircu sas3ircu hpssacli ssacli perccli \
|
|
multipath pvs vgs lvs lvdisplay \
|
|
xfs_info dumpe2fs btrfs \
|
|
fio iperf iperf3 sysbench bonnie++ iozone \
|
|
iostat mpstat pidstat sar vmstat \
|
|
top htop atop glances dstat nethogs iftop iotop smem pmap slabtop \
|
|
perf strace lsof fatrace \
|
|
dig host nslookup \
|
|
chronyc ntpq timedatectl \
|
|
nft iptables iptables-save ipset ufw \
|
|
jq bc awk smartctl nvme \
|
|
apt dpkg
|
|
do
|
|
p="$(command -v "$t" 2>/dev/null)"
|
|
if [ -n "$p" ]; then
|
|
v=""
|
|
case "$t" in
|
|
zpool|zfs) v="$(timeout 3 "$t" version 2>/dev/null | head -n 2 | tr '\n' ' ')" ;;
|
|
iperf3|fio|sysbench|smartctl|nvme|tuned-adm|ethtool|cpupower)
|
|
v="$(timeout 3 "$t" --version 2>&1 | head -n 1)" ;;
|
|
pveversion) v="$(timeout 3 "$t" 2>/dev/null)" ;;
|
|
storcli64|storcli|MegaCli64|MegaCli|arcconf|sas2ircu|sas3ircu|ssacli|hpssacli)
|
|
v="(storage HBA CLI - present)" ;;
|
|
esac
|
|
printf '%-22s %-8s %-30s %s\n' "$t" "yes" "$p" "$v"
|
|
else
|
|
printf '%-22s %-8s %-30s %s\n' "$t" "no" "-" "-"
|
|
fi
|
|
done
|
|
EOF
|
|
|
|
cmdsh "Debian packages relevant to perf/storage/net" <<'EOF'
|
|
dpkg -l 2>/dev/null | awk '/^ii/ && ($2 ~ /^(proxmox-|pve-|zfs|zfsutils|nfs|nfs-kernel-server|nfs-common|open-iscsi|ceph|tuned|irqbalance|sysstat|smartmontools|nvme-cli|ethtool|bridge-utils|ifenslave|fio|iperf3|sysbench|glances|htop|atop|lm-sensors|numactl|cpufrequtils|linux-image-|pve-kernel|qemu|libvirt|corosync|pacemaker)/) {print $2" "$3}' | head -n 80
|
|
EOF
|
|
|
|
cmdsh "kernel modules available (relevant to perf)" <<'EOF'
|
|
for m in kvm_intel kvm_amd \
|
|
ixgbe i40e i40iw ice igb igc mlx4_en mlx5_core mlx4_core bnx2x bnx2 tg3 be2net qed qede virtio_net \
|
|
nvme nvme-tcp nvme-fabrics sd_mod sr_mod \
|
|
zfs spl znvpair zcommon zunicode zavl icp zlua \
|
|
megaraid_sas mpt3sas hpsa aacraid smartpqi isci \
|
|
dm_multipath multipath \
|
|
tcp_dctcp tcp_bbr tcp_cubic tcp_htcp \
|
|
8021q bonding bridge br_netfilter \
|
|
vhost_net vhost_scsi target_core_mod \
|
|
nfsv4 nfsv3 rpcsec_gss_krb5
|
|
do
|
|
modinfo "$m" >/dev/null 2>&1 && printf 'available : %s (%s)\n' "$m" "$(modinfo -F version "$m" 2>/dev/null || echo built-in)"
|
|
done
|
|
EOF
|
|
|
|
cmdsh "currently loaded modules (filtered)" <<'EOF'
|
|
lsmod | awk 'NR==1 || $1 ~ /^(kvm|virtio|vhost|ixgb|i40e|ice|igb|mlx|bnx|tg3|be2|qed|nvme|zfs|spl|z|dm_|megaraid|mpt|hpsa|aacraid|smartpqi|8021q|bonding|bridge|nfsv|tcp_|rpc|sunrpc|loop)/{print}'
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 2. OS / DISTRO / UPTIME
|
|
# ===========================================================================
|
|
hdr "2. OS / DISTRO / UPTIME"
|
|
filecat "os-release" /etc/os-release
|
|
cmd "debian version" cat /etc/debian_version
|
|
cmd "uptime" uptime
|
|
cmd "uptime raw (since boot)" cat /proc/uptime
|
|
cmd "last reboot" last -x reboot 2>/dev/null | head -n 5
|
|
cmd "last logins" last 2>/dev/null | head -n 10
|
|
cmd "uname" uname -a
|
|
cmd "/proc/cmdline (boot args)" cat /proc/cmdline
|
|
|
|
# ===========================================================================
|
|
# 3. HARDWARE INVENTORY
|
|
# ===========================================================================
|
|
hdr "3. HARDWARE INVENTORY"
|
|
cmd "dmidecode -t system" dmidecode -t system
|
|
cmd "dmidecode -t baseboard" dmidecode -t baseboard
|
|
cmd "dmidecode -t bios" dmidecode -t bios
|
|
cmd "dmidecode -t chassis" dmidecode -t chassis
|
|
cmd "dmidecode -t processor" dmidecode -t processor
|
|
cmd "dmidecode -t memory" dmidecode -t memory
|
|
cmd "dmidecode -t cache" dmidecode -t cache
|
|
cmd "dmidecode -t connector" dmidecode -t connector
|
|
cmd "dmidecode -t slot" dmidecode -t slot
|
|
cmd "lscpu" lscpu
|
|
cmd "lscpu cache" lscpu -C
|
|
cmd "numactl --hardware" numactl --hardware
|
|
cmd "numactl --show" numactl --show
|
|
cmd "lspci -nnvv (verbose)" lspci -nnvv
|
|
cmd "lspci -tv (tree)" lspci -tv
|
|
cmd "lspci numa nodes" cat /sys/bus/pci/devices/*/numa_node 2>/dev/null | sort | uniq -c
|
|
cmd "lsusb" lsusb
|
|
cmd "lshw -short" lshw -short 2>/dev/null
|
|
dirlist "PCIe AER errors" /sys/bus/pci/devices/*/aer_dev_correctable
|
|
cmdsh "PCIe AER summary (errors matter for NIC/NVMe stability)" <<'EOF'
|
|
found=0
|
|
for d in /sys/bus/pci/devices/*; do
|
|
[ -d "$d" ] || continue
|
|
addr="$(basename "$d")"
|
|
class="$(cat "$d/class" 2>/dev/null)"
|
|
case "$class" in
|
|
0x020000|0x010802|0x010700) ;; # ethernet, nvme, raid
|
|
*) continue ;;
|
|
esac
|
|
for f in "$d"/aer_dev_*; do
|
|
[ -r "$f" ] || continue
|
|
val="$(cat "$f" 2>/dev/null)"
|
|
[ "$val" = "0" ] && continue
|
|
[ "$val" = "" ] && continue
|
|
printf '%s %s = %s\n' "$addr" "$(basename "$f")" "$val"
|
|
found=1
|
|
done
|
|
done
|
|
[ "$found" = "0" ] && echo " no AER errors on NIC/NVMe/RAID devices"
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 4. CPU POWER / IDLE / MICROCODE
|
|
# ===========================================================================
|
|
hdr "4. CPU POWER / FREQUENCY / IDLE STATES"
|
|
|
|
cmdsh "scaling_governor per cpu (counts)" <<'EOF'
|
|
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
|
[ -e "$c" ] || continue
|
|
cat "$c"
|
|
done | sort | uniq -c
|
|
EOF
|
|
|
|
cmdsh "scaling_driver / available governors / freq limits" <<'EOF'
|
|
for f in /sys/devices/system/cpu/cpu0/cpufreq/*; do
|
|
[ -r "$f" ] || continue
|
|
[ -d "$f" ] && continue
|
|
printf '%-40s = %s\n' "$(basename "$f")" "$(cat "$f" 2>/dev/null)"
|
|
done
|
|
EOF
|
|
|
|
cmdsh "EPP / EPB / intel_pstate" <<'EOF'
|
|
echo "intel_pstate status: $(cat /sys/devices/system/cpu/intel_pstate/status 2>/dev/null)"
|
|
echo "intel_pstate no_turbo: $(cat /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null)"
|
|
echo "intel_pstate turbo_pct: $(cat /sys/devices/system/cpu/intel_pstate/turbo_pct 2>/dev/null)"
|
|
echo "intel_pstate max_perf_pct: $(cat /sys/devices/system/cpu/intel_pstate/max_perf_pct 2>/dev/null)"
|
|
echo "intel_pstate min_perf_pct: $(cat /sys/devices/system/cpu/intel_pstate/min_perf_pct 2>/dev/null)"
|
|
echo "EPB (energy_perf_bias) per cpu:"
|
|
for c in /sys/devices/system/cpu/cpu*/power/energy_perf_bias; do
|
|
[ -r "$c" ] || continue
|
|
printf ' %s : %s\n' "$(echo "$c" | cut -d/ -f5)" "$(cat "$c" 2>/dev/null)"
|
|
done | sort | uniq -c
|
|
echo "EPP (energy_performance_preference) per cpu:"
|
|
for c in /sys/devices/system/cpu/cpu*/cpufreq/energy_performance_preference; do
|
|
[ -r "$c" ] || continue
|
|
printf ' %s : %s\n' "$(echo "$c" | cut -d/ -f5)" "$(cat "$c" 2>/dev/null)"
|
|
done | sort | uniq -c
|
|
echo "available EPP:"
|
|
cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_available_preferences 2>/dev/null
|
|
EOF
|
|
|
|
cmd "cpupower frequency-info" cpupower frequency-info 2>/dev/null
|
|
cmd "turbostat snapshot" timeout 3 turbostat --quiet --show CPU,Core,Bzy_MHz,IRQ,Busy%,Bzy_MHz,TSC_MHz,CoreTmp,PkgTmp sleep 1 2>/dev/null
|
|
cmd "x86_energy_perf_policy -r" x86_energy_perf_policy -r 2>/dev/null
|
|
|
|
cmdsh "C-states (idle) enabled per cpu0" <<'EOF'
|
|
for s in /sys/devices/system/cpu/cpu0/cpuidle/state*; do
|
|
[ -d "$s" ] || continue
|
|
printf ' %s : %s disabled=%s latency=%sus residency=%sus\n' \
|
|
"$(cat "$s/name" 2>/dev/null)" \
|
|
"$(cat "$s/desc" 2>/dev/null)" \
|
|
"$(cat "$s/disable" 2>/dev/null)" \
|
|
"$(cat "$s/latency" 2>/dev/null)" \
|
|
"$(cat "$s/time" 2>/dev/null)"
|
|
done
|
|
EOF
|
|
|
|
cmd "THP enabled" cat /sys/kernel/mm/transparent_hugepage/enabled
|
|
cmd "THP defrag" cat /sys/kernel/mm/transparent_hugepage/defrag
|
|
cmd "THP khugepaged pages_to_scan" cat /sys/kernel/mm/transparent_hugepage/khugepaged/pages_to_scan
|
|
cmd "THP khugepaged defrag" cat /sys/kernel/mm/transparent_hugepage/khugepaged/defrag
|
|
cmd "microcode" dmesg 2>/dev/null | grep -i microcode | tail -n 5
|
|
cmd "cpu MHz (live snapshot)" grep -c '^processor' /proc/cpuinfo; head -n 30 /proc/cpuinfo
|
|
|
|
# ===========================================================================
|
|
# 5. MEMORY / NUMA / SWAP
|
|
# ===========================================================================
|
|
hdr "5. MEMORY / NUMA / SWAP"
|
|
cmd "free -h" free -h
|
|
cmd "free -b" free -b
|
|
cmd "swapon" swapon --show
|
|
cmd "/proc/meminfo highlights" grep -E 'MemTotal|MemFree|MemAvailable|Buffers|^Cached|SwapTotal|SwapFree|Hugepagesize|HugePages_Total|HugePages_Free|AnonHugePages|DirectMap|Slab|SReclaimable|SUnreclaim|Dirty|Writeback|AnonPages|Mapped|Shmem|KernelStack|PageTables|NFS_Unstable' /proc/meminfo
|
|
cmd "numa stat" cat /proc/sys/vm/numa_stat 2>/dev/null
|
|
cmd "numa_balancing" cat /proc/sys/kernel/numa_balancing 2>/dev/null
|
|
cmd "zone reclaim" cat /proc/sys/vm/zone_reclaim_mode 2>/dev/null
|
|
cmd "slab top (top 20)" slabtop -o -s c 2>/dev/null | head -n 25
|
|
cmd "kswapd stats" grep -E 'kswapd|pgmigrate|pgmigrate_failed|compact_' /proc/vmstat 2>/dev/null | head -n 30
|
|
|
|
# ===========================================================================
|
|
# 6. KERNEL TUNABLES
|
|
# ===========================================================================
|
|
hdr "6. KERNEL TUNABLES / SYSCTL"
|
|
|
|
cmdsh "vm.* tunables" <<'EOF'
|
|
sysctl -a 2>/dev/null | grep -E '^vm\.(dirty_(ratio|background_ratio|bytes|background_bytes|expire_centisecs|writeback_centisecs)|swappiness|vfs_cache_pressure|min_free_kbytes|watermark_scale_factor|overcommit_memory|overcommit_ratio|nr_hugepages|nr_overcommit_hugepages|zone_reclaim_mode|compact_unevictable_allowed|stat_interval|admin_reserve_kbytes|user_reserve_kbytes)'
|
|
EOF
|
|
|
|
cmdsh "fs.* tunables" <<'EOF'
|
|
sysctl -a 2>/dev/null | grep -E '^fs\.(file-max|file-nr|aio-max-nr|aio-nr|nr_open|pipe-max-size|epoll/max_user_watches|lease-break-time|mqueue|max_user_ns)'
|
|
EOF
|
|
|
|
cmdsh "kernel.* perf-relevant" <<'EOF'
|
|
sysctl -a 2>/dev/null | grep -E '^kernel\.(sched_(child_runs_first|migration_cost|domain|rt_period|rt_runtime|idle)|numa_balancing|watchdog|nmi_watchdog|hung_task|randomize_va_space|max_pid|threads-max|sched_time_avg)'
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 7. NETWORK SYSCTL / TCP
|
|
# ===========================================================================
|
|
hdr "7. NETWORK SYSCTL / TCP"
|
|
cmdsh "net.core.* tunables" <<'EOF'
|
|
sysctl -a 2>/dev/null | grep -E '^net\.core\.(rmem_default|rmem_max|wmem_default|wmem_max|optmem_max|somaxconn|netdev_max_backlog|netdev_budget|netdev_budget_usecs|netdev_tstamp_prequeue|rps_flow_cnt|busy_poll|busy_read|default_qdisc|dev_weight|xfrm)'
|
|
EOF
|
|
|
|
cmdsh "net.ipv4.tcp.* tunables" <<'EOF'
|
|
sysctl -a 2>/dev/null | grep -E '^net\.ipv4\.tcp_(rmem|wmem|congestion_control|available_congestion_control|mtu_probing|slow_start_after_idle|no_metrics_save|timestamps|window_scaling|sack|low_latency|fastopen|ecn|ecn_fallback|syncookies|max_syn_backlog|tw_reuse|fin_timeout|keepalive_time|keepalive_probes|keepalive_intvl|thp_linear_timeouts|pacing|pacing_ss_ratio|pacing_ca_ratio|no_ssthresh_metrics_save|md5sig|abort_on_overflow|buffer_split)'
|
|
EOF
|
|
|
|
cmdsh "other net.* sysctls" <<'EOF'
|
|
sysctl -a 2>/dev/null | grep -E '^net\.(ipv4\.ip_(local_port_range|local_reserved_ports|no_pmtu_disc|forwarding)|ipv4\.conf\.all\.(rp_filter|accept_redirects|send_redirects)|ipv4\.conf\.default\.forwarding|core\.netdev_budget|ipv6\.conf\.all\.disable_ipv6)'
|
|
EOF
|
|
|
|
dirlist "sysctl.d" /etc/sysctl.d
|
|
filecat "/etc/sysctl.conf" /etc/sysctl.conf
|
|
cmdsh "non-default sysctl values (sorted, filtered)" <<'EOF'
|
|
# Show only sysctls that differ from running state vs an empty config;
|
|
# too noisy to dump all of /proc/sys. Just show /etc/sysctl.d content.
|
|
for f in /etc/sysctl.d/*.conf /etc/sysctl.conf; do
|
|
[ -r "$f" ] || continue
|
|
echo "=== $f ==="
|
|
grep -vE '^\s*#|^\s*$' "$f" 2>/dev/null
|
|
done
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 8. tuned / irqbalance / udev
|
|
# ===========================================================================
|
|
hdr "8. tuned / irqbalance / udev"
|
|
cmd "tuned-adm active" tuned-adm active 2>/dev/null
|
|
cmd "tuned-adm list" tuned-adm list 2>/dev/null
|
|
cmd "systemctl is-active tuned" systemctl is-active tuned 2>/dev/null
|
|
cmd "irqbalance status" systemctl is-active irqbalance 2>/dev/null
|
|
filecat "irqbalance config" /etc/default/irqbalance /etc/irqbalance.conf
|
|
dirlist "modprobe.d" /etc/modprobe.d
|
|
dirlist "udev rules.d" /etc/udev/rules.d
|
|
filecat "modprobe.d files" /etc/modprobe.d/*.conf
|
|
cmdsh "module params: kvm_intel" <<'EOF'
|
|
for p in /sys/module/kvm_intel/parameters/*; do
|
|
[ -e "$p" ] && printf ' %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
|
|
done
|
|
EOF
|
|
cmdsh "module params: kvm_amd" <<'EOF'
|
|
for p in /sys/module/kvm_amd/parameters/*; do
|
|
[ -e "$p" ] && printf ' %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
|
|
done
|
|
EOF
|
|
cmdsh "module params: relevant net drivers" <<'EOF'
|
|
for mod in i40e ixgbe mlx5_core igb ice bnx2x virtio_net; do
|
|
[ -d "/sys/module/$mod/parameters" ] || continue
|
|
echo "=== $mod ==="
|
|
for p in /sys/module/$mod/parameters/*; do
|
|
[ -e "$p" ] && printf ' %s = %s\n' "$(basename "$p")" "$(cat "$p" 2>/dev/null)"
|
|
done
|
|
done
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 9. PROXMOX / CLUSTER
|
|
# ===========================================================================
|
|
hdr "9. PROXMOX VERSION & CLUSTER"
|
|
cmd "pveversion" pveversion
|
|
cmd "pveversion -v" pveversion -v
|
|
cmd "pvecm status" pvecm status
|
|
cmd "pvecm nodes" pvecm nodes
|
|
cmd "pvecm expected" pvecm expected 2>/dev/null
|
|
cmd "pveperf" timeout 15 pveperf 2>/dev/null
|
|
filecat "corosync.conf" /etc/pve/corosync.conf
|
|
filecat "datacenter.cfg" /etc/pve/datacenter.cfg
|
|
filecat "storage.cfg" /etc/pve/storage.cfg
|
|
filecat "user.cfg" /etc/pve/user.cfg 2>/dev/null
|
|
filecat "domains.cfg" /etc/pve/domains.cfg 2>/dev/null
|
|
cmdsh "cluster resources (compact)" <<'EOF'
|
|
pvesh get /cluster/resources --type node --output-format json 2>/dev/null \
|
|
| jq -r '.[] | "\(.node)\tcpu=\(.cpu)\tmem=\(.mem)\tmaxmem=\(.maxmem)\tmaxdisk=\(.maxdisk)\tstatus=\(.status)\tssl_fingerprint=\(.ssl_fingerprint)"' 2>/dev/null \
|
|
|| pvesh get /cluster/resources --type node 2>/dev/null
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 10. NETWORK: INTERFACES
|
|
# ===========================================================================
|
|
hdr "10. NETWORK: INTERFACES"
|
|
cmd "ip -d link" ip -d link
|
|
cmd "ip -o link" ip -o link
|
|
cmd "ip -o -4 addr" ip -o -4 addr
|
|
cmd "ip -o -6 addr (link-local filter)" ip -o -6 addr 2>/dev/null | grep -v 'scope link'
|
|
cmd "ip addr" ip addr
|
|
cmd "ip route" ip route
|
|
cmd "ip rule" ip rule
|
|
cmd "ip -s -s link (counters/errors)" ip -s -s link
|
|
cmd "ip neigh" ip neigh
|
|
cmd "/proc/net/dev (counters)" cat /proc/net/dev
|
|
cmd "ip -6 route (brief)" ip -6 route 2>/dev/null | head -n 20
|
|
|
|
# Build NETIFS list once, export for use in heredoc snippets
|
|
NETIFS=""
|
|
for _n in /sys/class/net/*; do
|
|
_n="$(basename "$_n")"
|
|
case "$_n" in
|
|
lo|sit0|ip6tnl0|gre0|gretap0|erspan0|bonding_masters|tailscale0|wg0|tunl0) continue ;;
|
|
esac
|
|
NETIFS="${NETIFS:+$NETIFS }$_n"
|
|
done
|
|
export NETIFS
|
|
|
|
cmdsh "sysfs interface attributes" <<'EOF'
|
|
for ifc in $NETIFS; do
|
|
d="/sys/class/net/$ifc"
|
|
[ -d "$d" ] || continue
|
|
printf '%-16s speed=%-8s duplex=%-8s mtu=%-6s carrier=%-3s tx_queue_len=%-5s qlen=%-5s type=%s\n' \
|
|
"$ifc" \
|
|
"$(cat "$d/speed" 2>/dev/null)" \
|
|
"$(cat "$d/duplex" 2>/dev/null)" \
|
|
"$(cat "$d/mtu" 2>/dev/null)" \
|
|
"$(cat "$d/carrier" 2>/dev/null)" \
|
|
"$(cat "$d/tx_queue_len" 2>/dev/null)" \
|
|
"$(cat "$d/tx_queue_len" 2>/dev/null)" \
|
|
"$(cat "$d/type" 2>/dev/null)"
|
|
done
|
|
EOF
|
|
|
|
# Per-interface deep ethtool dump
|
|
cmdsh "ethtool per-interface deep dump" <<'EOF'
|
|
for ifc in $NETIFS; do
|
|
[ -e "/sys/class/net/$ifc" ] || continue
|
|
echo ""
|
|
echo "----- ethtool: $ifc -----"
|
|
echo "[driver/version/firmware/bus-info -i]"
|
|
ethtool -i "$ifc" 2>&1
|
|
echo "[settings - speed/duplex/autoneg/port]"
|
|
ethtool "$ifc" 2>&1
|
|
echo "[channels -l (RX/TX/combined queue counts)]"
|
|
ethtool -l "$ifc" 2>&1
|
|
echo "[coalesce -c (interrupt moderation)]"
|
|
ethtool -c "$ifc" 2>&1
|
|
echo "[ring -g (descriptor counts)]"
|
|
ethtool -g "$ifc" 2>&1
|
|
echo "[pause / flow-control -a]"
|
|
ethtool -a "$ifc" 2>&1
|
|
echo "[offload features -k]"
|
|
ethtool -k "$ifc" 2>&1
|
|
echo "[private flags --show-priv-flags]"
|
|
ethtool --show-priv-flags "$ifc" 2>&1
|
|
echo "[ntuple/RSS -n]"
|
|
ethtool -n "$ifc" 2>&1 | head -n 40
|
|
echo "[SFP/DOM transceiver -m]"
|
|
ethtool -m "$ifc" 2>&1
|
|
echo "[statistics -S (key error/drop counters only)]"
|
|
ethtool -S "$ifc" 2>/dev/null \
|
|
| grep -iE 'drop|discard|error|crc|miss|pause|fifo|no_buff|rx_no_|_err|fw_|_bad|oversiz|undersize|frag|jabber|length_err|alignment' \
|
|
| head -n 60
|
|
echo "[statistics -S (throughput counters)]"
|
|
ethtool -S "$ifc" 2>/dev/null \
|
|
| grep -iE 'rx_(octets|bytes|packets)|tx_(octets|bytes|packets)|rx_(unicast|multicast|broadcast)|tx_(unicast|multicast|broadcast)' \
|
|
| head -n 30
|
|
done
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 11. NETWORK: BRIDGES / BONDS / VLANS
|
|
# ===========================================================================
|
|
hdr "11. NETWORK: BRIDGES / BONDS / VLANS"
|
|
cmd "bridge link" bridge link 2>/dev/null
|
|
cmd "bridge -d link" bridge -d link 2>/dev/null
|
|
cmd "bridge vlan" bridge vlan 2>/dev/null
|
|
cmd "bridge vlan show" bridge vlan show 2>/dev/null
|
|
cmd "brctl show" brctl show 2>/dev/null
|
|
filecat "bonding state" /proc/net/bonding/bond*
|
|
cmd "vlan config" cat /proc/net/vlan/config 2>/dev/null
|
|
dirlist "/proc/net/vlan" /proc/net/vlan
|
|
filecat "/etc/network/interfaces" /etc/network/interfaces
|
|
cmdsh "interfaces.d (per file)" <<'EOF'
|
|
for f in /etc/network/interfaces.d/*; do
|
|
[ -r "$f" ] || continue
|
|
echo "--- $f ---"
|
|
cat "$f"
|
|
done
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 12. NETWORK: IRQ / RPS / XPS / aRFS / SOFTNET
|
|
# ===========================================================================
|
|
hdr "12. NETWORK: IRQ / RPS / XPS / aRFS / SOFTNET"
|
|
cmd "/proc/interrupts" cat /proc/interrupts
|
|
|
|
cmdsh "NIC IRQ affinities (which CPUs handle which queue)" <<'EOF'
|
|
for ifc in $NETIFS; do
|
|
[ -d "/sys/class/net/$ifc/device/msi_irqs" ] || continue
|
|
echo "=== $ifc IRQ list (first 40) ==="
|
|
ls "/sys/class/net/$ifc/device/msi_irqs" 2>/dev/null | head -n 40
|
|
done
|
|
EOF
|
|
|
|
cmdsh "RPS masks (non-zero only)" <<'EOF'
|
|
for q in /sys/class/net/*/queues/rx-*/rps_cpus; do
|
|
[ -e "$q" ] || continue
|
|
val=$(cat "$q")
|
|
[ "$val" != "0" ] && printf ' %s = %s\n' "$q" "$val"
|
|
done | head -n 60
|
|
EOF
|
|
|
|
cmdsh "XPS masks (non-zero only)" <<'EOF'
|
|
for q in /sys/class/net/*/queues/tx-*/xps_cpus; do
|
|
[ -e "$q" ] || continue
|
|
val=$(cat "$q")
|
|
[ "$val" != "0" ] && printf ' %s = %s\n' "$q" "$val"
|
|
done | head -n 60
|
|
EOF
|
|
|
|
cmdsh "RFS flow counts" <<'EOF'
|
|
for q in /sys/class/net/*/queues/rx-*/rps_flow_cnt; do
|
|
[ -e "$q" ] || continue
|
|
printf ' %s = %s\n' "$q" "$(cat "$q")"
|
|
done | head -n 60
|
|
EOF
|
|
|
|
cmd "/proc/net/softnet_stat (drops/backlog)" cat /proc/net/softnet_stat
|
|
cmdsh "softnet_stat decoded" <<'EOF'
|
|
# columns: TOTAL DROPPED DROPPED_BY_NETRX TIME_SQUEEZED CPU_COLLISIONS RPS_RECEIVED_FLOW_LIMIT
|
|
awk '{ printf "cpu%-2d total=%-8s dropped=%-6s time_squeezed=%-6s cpu_collision=%-4s rps_received=%-6s flow_limit=%s\n",
|
|
NR-1, $1, $2, $4, $5, $6, $7 }' /proc/net/softnet_stat
|
|
EOF
|
|
|
|
cmdsh "netstat drop counters (significant ones)" <<'EOF'
|
|
nstat -az 2>/dev/null | grep -iE '(Drop|Retrans|Fail|Abort|Overflow|Loss|ListenDrop|EmbryonicRsts|InErrs|OutErrs)' || \
|
|
netstat -s 2>/dev/null | grep -iE '(drop|retrans|fail|abort|overflow|loss|listen|error)' | head -n 60
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 13. NETWORK: SOCKET / TCP STATE
|
|
# ===========================================================================
|
|
hdr "13. NETWORK: SOCKET / TCP STATE"
|
|
cmd "ss -s" ss -s
|
|
cmd "ss -t -m (top TCP with skmem/cwnd/rto)" ss -tno state established '( dport != :22 and sport != :22 )' -m 2>/dev/null | head -n 40
|
|
cmd "ss -ti (TCP internals)" ss -ti 2>/dev/null | head -n 80
|
|
cmd "/proc/net/snmp (TCP summary)" grep -A1 '^Tcp:' /proc/net/snmp 2>/dev/null
|
|
cmd "TCP retrans stats" grep -A1 '^Tcp:' /proc/net/snmp 2>/dev/null | tail -n 1 | awk '{print "RetransSegs="$13}'
|
|
cmd "nstat TCP summary" nstat -az Tcp_RetransSegs Tcp_InErrs Tcp_OutRsts Tcp_ActiveOpens Tcp_PassiveOpens 2>/dev/null
|
|
|
|
# ===========================================================================
|
|
# 14. STORAGE: BLOCK DEVICES
|
|
# ===========================================================================
|
|
hdr "14. STORAGE: BLOCK DEVICES"
|
|
cmd "lsblk (key cols)" lsblk -o NAME,SIZE,TYPE,MOUNTPOINT,FSTYPE,MODEL,SERIAL,STATE,ROTA,DISC-GRAN,DISC-MAX
|
|
cmd "lsblk -t" lsblk -t
|
|
cmd "lsblk -O (everything, capped)" lsblk -O | head -n 200
|
|
cmd "blkid" blkid 2>/dev/null
|
|
cmd "/proc/partitions" cat /proc/partitions
|
|
cmd "fdisk -l" fdisk -l 2>/dev/null
|
|
cmd "parted -l" parted -l 2>/dev/null
|
|
|
|
cmdsh "block device queue settings (per device)" <<'EOF'
|
|
for d in $(ls /sys/block/ 2>/dev/null | grep -vE '^(loop|ram|md|dm-|sr|fd)'); do
|
|
q="/sys/block/$d/queue"
|
|
[ -d "$q" ] || continue
|
|
printf '----- %s -----\n' "$d"
|
|
for k in scheduler rotational nr_requests read_ahead_kb max_sectors_kb max_hw_sectors_kb \
|
|
logical_block_size physical_block_size discard_granularity discard_max_hw_bytes \
|
|
rq_affinity nomerges wbt_lat_usec write_cache stable_writes io_poll io_poll_delay \
|
|
fua rot_stable_writes add_random zoned dma_alignment iostats;
|
|
do
|
|
f="$q/$k"
|
|
[ -r "$f" ] && printf ' %-22s = %s\n' "$k" "$(cat "$f" 2>/dev/null)"
|
|
done
|
|
# NVMe-specific
|
|
for f in /sys/block/$d/device/model /sys/block/$d/device/serial /sys/block/$d/device/transport /sys/block/$d/device/queue_count; do
|
|
[ -r "$f" ] && printf ' %s = %s\n' "$(echo "$f" | sed "s|/sys/block/$d/device/||")" "$(cat "$f")"
|
|
done
|
|
done
|
|
EOF
|
|
|
|
cmd "df -h (filesystem usage)" df -h
|
|
cmd "df -i (inode usage)" df -i
|
|
|
|
# ===========================================================================
|
|
# 15. STORAGE: MOUNTS & FILESYSTEMS
|
|
# ===========================================================================
|
|
hdr "15. STORAGE: MOUNTS & FILESYSTEMS"
|
|
cmd "mount (filtered)" mount | grep -vE 'proc|sysfs|cgroup|tmpfs|devpts|mqueue|pstore|bpf|tracefs|debugfs|configfs|fusectl|securityfs|hugetlbfs|autofs|systemd|mtd|rpc_pipefs|binfmt'
|
|
cmd "findmnt (tree)" findmnt -t nfs,nfs4,zfs,ext4,xfs,btrfs 2>/dev/null
|
|
filecat "/etc/fstab" /etc/fstab
|
|
cmdsh "ext4 reserved block % (per fs)" <<'EOF'
|
|
for dev in $(awk '$3=="ext4"{print $1}' /proc/mounts 2>/dev/null); do
|
|
echo "=== $dev ==="
|
|
dumpe2fs -h "$dev" 2>/dev/null | grep -E 'Reserved|Block count|Free blocks|Block size|Mount count|Maximum mount|Last'
|
|
done
|
|
EOF
|
|
cmdsh "xfs info (per fs)" <<'EOF'
|
|
for mp in $(awk '$3=="xfs"{print $2}' /proc/mounts 2>/dev/null); do
|
|
echo "=== $mp ==="
|
|
xfs_info "$mp" 2>/dev/null
|
|
done
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 16. STORAGE: NFS (client + server) - critical for this cluster
|
|
# ===========================================================================
|
|
hdr "16. STORAGE: NFS (CLIENT + SERVER)"
|
|
cmd "nfsstat -m (mount options)" nfsstat -m 2>/dev/null
|
|
cmd "nfsstat -c (client stats)" nfsstat -c 2>/dev/null
|
|
cmd "nfsstat -s (server stats)" nfsstat -s 2>/dev/null
|
|
cmd "nfsstat -234 (version stats)" nfsstat -234 2>/dev/null
|
|
cmd "showmount -e (exports)" showmount -e 2>/dev/null
|
|
cmd "/proc/net/rpc/nfs" cat /proc/net/rpc/nfs 2>/dev/null | head -n 60
|
|
cmd "/proc/net/rpc/nfsd" cat /proc/net/rpc/nfsd 2>/dev/null | head -n 60
|
|
filecat "exports" /etc/exports /etc/exports.d/*.exports
|
|
filecat "nfs.conf" /etc/nfs.conf /etc/nfs.conf.d/*.conf
|
|
filecat "nfs defaults" /etc/default/nfs-kernel-server /etc/default/nfs-common
|
|
filecat "idmapd.conf" /etc/idmapd.conf
|
|
|
|
cmd "pvesm status" pvesm status
|
|
cmdsh "pvesm list per storage" <<'EOF'
|
|
for s in $(pvesm status 2>/dev/null | awk 'NR>1 && $3=="active"{print $1}'); do
|
|
echo "--- storage: $s ---"
|
|
pvesm list "$s" 2>/dev/null | head -n 30
|
|
echo "--- pvesm config $s ---"
|
|
pvesm config "$s" 2>/dev/null
|
|
done
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 17. STORAGE: LVM
|
|
# ===========================================================================
|
|
hdr "17. STORAGE: LVM"
|
|
cmd "pvs" pvs -v
|
|
cmd "vgs" vgs -v
|
|
cmd "lvs" lvs -o +lv_layout,role,pool_lv,data_percent,metadata_percent --units b
|
|
cmd "pvdisplay" pvdisplay
|
|
cmd "vgdisplay" vgdisplay
|
|
cmd "lvdisplay" lvdisplay
|
|
filecat "lvm/lvm.conf (filters only)" /etc/lvm/lvm.conf
|
|
cmdsh "lvm.conf active filter" <<'EOF'
|
|
grep -E '^\s*(filter|global_filter|use_devicesfile|obtain_device_list_from_udev)' /etc/lvm/lvm.conf 2>/dev/null
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 18. STORAGE: MULTIPATH
|
|
# ===========================================================================
|
|
hdr "18. STORAGE: MULTIPATH"
|
|
cmd "multipath -ll" multipath -ll
|
|
cmd "multipath -v3 -ll (verbose, capped)" multipath -v3 -ll 2>/dev/null | head -n 100
|
|
filecat "multipath.conf" /etc/multipath.conf
|
|
cmd "systemctl is-active multipathd" systemctl is-active multipathd 2>/dev/null
|
|
|
|
# ===========================================================================
|
|
# 19. STORAGE: ZFS
|
|
# ===========================================================================
|
|
hdr "19. STORAGE: ZFS"
|
|
if command -v zpool >/dev/null 2>&1; then
|
|
cmd "zfs version (userland)" zfs version 2>/dev/null
|
|
cmd "zpool version" zpool version 2>/dev/null
|
|
cmd "zpool list -v" zpool list -v
|
|
cmd "zpool status -v" zpool status -v
|
|
cmdsh "zpool key properties" <<'EOF'
|
|
zpool get name,size,alloc,free,expandsize,fragmentation,capacity,health,ashift,autotrim,autoreplace,cachefile,comment,listsnapshots,multihost,version,bootfs,delegation,failmode,freeing,leaked,readonly
|
|
EOF
|
|
cmdsh "zfs per-dataset key properties" <<'EOF'
|
|
zfs get -r -o name,property,value,source \
|
|
compression,atime,recordsize,volblocksize,sync,primarycache,secondarycache,checksum,xattr,acltype,aclinherit,copies,logbias,refreservation,reservation,quota,used,avail,refer,mountpoint,mounted,canmount,relative_path,snapdir,version,primarycache,redundant_metadata,encryption 2>/dev/null
|
|
EOF
|
|
cmd "ARC stats" cat /proc/spl/kstat/zfs/arcstats
|
|
cmd "dmu_tx" cat /proc/spl/kstat/zfs/dmu_tx 2>/dev/null
|
|
cmd "zfetchstats" cat /proc/spl/kstat/zfs/zfetchstats 2>/dev/null
|
|
cmd "vdev_cache_stats" cat /proc/spl/kstat/zfs/vdev_cache_stats 2>/dev/null
|
|
cmd "abdstats" cat /proc/spl/kstat/zfs/abdstats 2>/dev/null
|
|
cmdsh "ZFS module parameters (perf-relevant)" <<'EOF'
|
|
for f in /sys/module/zfs/parameters/*; do
|
|
[ -r "$f" ] || continue
|
|
name="$(basename "$f")"
|
|
case "$name" in
|
|
zfs_arc_*|zfs_abd_*|zfs_dbuf_*|zfs_txg_*|zfs_vdev_*|zfs_sync_*|zfs_dirty_*|zfs_*io_*|zvol_*|l2arc_*|zfs_prefetch_*|zfs_*max_active|zfs_*_taskq*) ;;
|
|
*) continue ;;
|
|
esac
|
|
printf ' %-50s = %s\n' "$name" "$(cat "$f")"
|
|
done
|
|
EOF
|
|
cmd "zpool iostat snapshot" zpool iostat 1 1 2>/dev/null
|
|
cmd "arc_summary (if present)" arc_summary 2>/dev/null | head -n 80
|
|
cmd "zpool events -v (recent)" zpool events -v 2>/dev/null | tail -n 50
|
|
cmdsh "zpool history (last 30 events)" <<'EOF'
|
|
zpool history -il 2>/dev/null | tail -n 30
|
|
EOF
|
|
else
|
|
printf '\nZFS not present on this host.\n' >> "$OUT"
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# 20. STORAGE: DRIVE HEALTH (SMART / NVMe)
|
|
# ===========================================================================
|
|
hdr "20. STORAGE: DRIVE HEALTH"
|
|
cmd "nvme list" nvme list 2>/dev/null
|
|
cmdsh "nvme smart-log per device" <<'EOF'
|
|
for n in /dev/nvme[0-9]*; do
|
|
[ -b "$n" ] || continue
|
|
echo "--- $n smart-log ---"
|
|
nvme smart-log "$n" 2>&1
|
|
echo "--- $n error-log (first 5) ---"
|
|
nvme error-log "$n" 2>&1 | head -n 80
|
|
done
|
|
EOF
|
|
cmdsh "nvme id-ctrl per device (key fields)" <<'EOF'
|
|
for n in /dev/nvme[0-9]*; do
|
|
[ -b "$n" ] || continue
|
|
echo "--- $n id-ctrl (filtered) ---"
|
|
nvme id-ctrl "$n" 2>&1 | grep -E '^(mn|sn|fn|fr|tnvmcap|vid|IEEE|cntlid|ver|rab|lpa|elpe|npdg|mdts|hmpre|hmmin|cap|frmw|lba|ws|awun|awupf|acwu|apsta)'
|
|
done
|
|
EOF
|
|
if command -v smartctl >/dev/null 2>&1; then
|
|
cmdsh "smartctl -a per disk" <<'EOF'
|
|
for d in $(ls /sys/block/ 2>/dev/null | grep -E '^(sd|vd)'); do
|
|
dev="/dev/$d"
|
|
[ -b "$dev" ] || continue
|
|
echo "--- smartctl -a $dev ---"
|
|
smartctl -a "$dev" 2>&1 | head -n 200
|
|
done
|
|
EOF
|
|
fi
|
|
|
|
# ===========================================================================
|
|
# 21. STORAGE: HBA / RAID CONTROLLERS
|
|
# ===========================================================================
|
|
hdr "21. STORAGE: HBA / RAID CONTROLLERS"
|
|
for cli in storcli64 storcli MegaCli64 MegaCli perccli; do
|
|
if command -v "$cli" >/dev/null 2>&1; then
|
|
printf '\n--- %s found ---\n' "$cli" >> "$OUT"
|
|
case "$cli" in
|
|
storcli*|perccli)
|
|
cmd "$cli /call show all" "$cli" /call show all
|
|
cmd "$cli /call/vall show all" "$cli" /call/vall show all 2>/dev/null
|
|
;;
|
|
MegaCli*)
|
|
cmd "MegaCli adapter info" "$cli" -AdpAllInfo -aALL
|
|
cmd "MegaCli physical drives" "$cli" -PDList -aALL
|
|
cmd "MegaCli logical drives" "$cli" -LDInfo -Lall -aALL
|
|
cmd "MegaCli BBUs" "$cli" -AdpBbuCmd -aALL
|
|
;;
|
|
esac
|
|
break
|
|
fi
|
|
done
|
|
for cli in sas2ircu sas3ircu; do
|
|
if command -v "$cli" >/dev/null 2>&1; then
|
|
cmd "$cli list" "$cli" list
|
|
cmd "$cli 0 display" "$cli" 0 display 2>/dev/null
|
|
fi
|
|
done
|
|
command -v arcconf >/dev/null 2>&1 && cmd "arcconf list" arcconf list
|
|
command -v ssacli >/dev/null 2>&1 && cmd "ssacli ctrl all show config" ssacli ctrl all show config
|
|
command -v hpssacli >/dev/null 2>&1 && cmd "hpssacli ctrl all show config" hpssacli ctrl all show config
|
|
|
|
# ===========================================================================
|
|
# 22. VIRTUAL MACHINES (qm)
|
|
# ===========================================================================
|
|
hdr "22. VIRTUAL MACHINES (qm)"
|
|
cmd "qm list" qm list
|
|
cmdsh "qm config per VM" <<'EOF'
|
|
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
|
|
echo "--- VM $vmid ---"
|
|
qm config "$vmid" 2>&1
|
|
echo ""
|
|
done
|
|
EOF
|
|
cmdsh "VM runtime status (qm guest agent if running)" <<'EOF'
|
|
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
|
|
echo "--- VM $vmid runtime ---"
|
|
qm guest cmd "$vmid" network-get-interfaces 2>/dev/null | head -n 40 || echo " [no agent or VM not running]"
|
|
done
|
|
EOF
|
|
dirlist "/etc/pve/qemu-server" /etc/pve/qemu-server
|
|
dirlist "/etc/pve/firewall" /etc/pve/firewall
|
|
|
|
# ===========================================================================
|
|
# 23. CONTAINERS (pct)
|
|
# ===========================================================================
|
|
hdr "23. CONTAINERS (pct)"
|
|
cmd "pct list" pct list
|
|
cmdsh "pct config per CT" <<'EOF'
|
|
for ctid in $(pct list 2>/dev/null | awk 'NR>1{print $1}'); do
|
|
echo "--- CT $ctid ---"
|
|
pct config "$ctid" 2>&1
|
|
echo ""
|
|
done
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 24. SERVICES / FAILED UNITS
|
|
# ===========================================================================
|
|
hdr "24. SERVICES / FAILED UNITS"
|
|
cmd "running services" systemctl list-units --type=service --state=running --no-pager
|
|
cmd "failed services" systemctl --failed --no-pager
|
|
cmd "enabled services" systemctl list-unit-files --state=enabled --no-pager | head -n 80
|
|
cmdsh "pve-* service status" <<'EOF'
|
|
systemctl status pve-cluster pvedaemon pveproxy pve-firewall pve-ha-crm pve-ha-lrm pvestatd pvebanner --no-pager 2>&1 | head -n 120
|
|
EOF
|
|
cmdsh "storage-related service status" <<'EOF'
|
|
systemctl status nfs-kernel-server nfs-server rpcbind rpc-statd zfs-import zfs-mount zfs-target smartd nvme-stat multipathd --no-pager 2>&1 | head -n 80
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 25. TIMERS / CRON (could cause load spikes during work hours)
|
|
# ===========================================================================
|
|
hdr "25. TIMERS / CRON"
|
|
cmd "systemctl list-timers" systemctl list-timers --all --no-pager | head -n 40
|
|
dirlist "/etc/cron.d" /etc/cron.d
|
|
dirlist "/etc/cron.daily" /etc/cron.daily
|
|
dirlist "/etc/cron.hourly" /etc/cron.hourly
|
|
dirlist "/etc/cron.weekly" /etc/cron.weekly
|
|
filecat "crontabs" /etc/crontab
|
|
cmdsh "root crontab" <<'EOF'
|
|
crontab -l 2>/dev/null || echo " [no root crontab]"
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 26. LIVE PERFORMANCE SNAPSHOT
|
|
# ===========================================================================
|
|
hdr "26. LIVE PERFORMANCE SNAPSHOT"
|
|
cmd "/proc/loadavg" cat /proc/loadavg
|
|
cmd "vmstat 1 3" vmstat 1 3
|
|
cmd "mpstat 1 3 (per-cpu)" mpstat 1 3 2>/dev/null
|
|
cmd "iostat -xz 1 3" iostat -xz 1 3 2>/dev/null
|
|
cmd "sar -dp 1 2" sar -dp 1 2 2>/dev/null
|
|
cmd "sar -n DEV 1 2" sar -n DEV 1 2 2>/dev/null
|
|
cmd "sar -n NFS,NFSD 1 2" sar -n NFS,NFSD 1 2 2>/dev/null
|
|
cmd "sar -n SOCK 1 1" sar -n SOCK 1 1 2>/dev/null
|
|
cmd "sar -n TCP,ETCP 1 2" sar -n TCP,ETCP 1 2 2>/dev/null
|
|
cmd "pidstat -d 1 2 (top disk)" pidstat -d 1 2 2>/dev/null | head -n 40
|
|
cmd "pidstat -u 1 2 (top cpu)" pidstat -u 1 2 2>/dev/null | head -n 40
|
|
cmd "top (single snapshot, top 30)" top -b -n 1 | head -n 35
|
|
cmd "zpool iostat 1 2" zpool iostat 1 2 2>/dev/null
|
|
cmd "arcstat 1 2" arcstat 1 2 2>/dev/null
|
|
|
|
# /proc/net/dev snapshot twice for delta
|
|
cmdsh "/proc/net/dev delta (1s)" <<'EOF'
|
|
awk '{print $1}' /proc/net/dev
|
|
echo "--- t0 ---"
|
|
cat /proc/net/dev
|
|
sleep 1
|
|
echo "--- t1 (1s later) ---"
|
|
cat /proc/net/dev
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 27. DNS / NTP / TIME
|
|
# ===========================================================================
|
|
hdr "27. DNS / NTP / TIME"
|
|
filecat "/etc/resolv.conf" /etc/resolv.conf
|
|
filecat "/etc/hosts" /etc/hosts
|
|
filecat "/etc/ntp.conf" /etc/ntp.conf 2>/dev/null
|
|
filecat "/etc/chrony/chrony.conf" /etc/chrony/chrony.conf 2>/dev/null
|
|
cmd "timedatectl" timedatectl 2>/dev/null
|
|
cmd "chronyc tracking" chronyc tracking 2>/dev/null
|
|
cmd "chronyc sources" chronyc sources 2>/dev/null
|
|
cmd "ntpq peers" ntpq -pn 2>/dev/null
|
|
|
|
# ===========================================================================
|
|
# 28. FIREWALL
|
|
# ===========================================================================
|
|
hdr "28. FIREWALL / SECURITY"
|
|
cmd "pve-firewall status" pve-firewall status 2>/dev/null
|
|
cmd "nft ruleset (count)" nft list ruleset 2>/dev/null | wc -l
|
|
cmd "nft ruleset (head)" nft list ruleset 2>/dev/null | head -n 60
|
|
cmd "iptables-save (legacy)" iptables-save 2>/dev/null | head -n 60
|
|
cmd "ipset list (names)" ipset list -n 2>/dev/null
|
|
cmd "apparmor status" aa-status 2>/dev/null | head -n 20
|
|
|
|
# ===========================================================================
|
|
# 29. TOP CONSUMERS (memory + disk + file handles)
|
|
# ===========================================================================
|
|
hdr "29. TOP CONSUMERS"
|
|
cmdsh "top 10 memory hogs" <<'EOF'
|
|
ps -eo pid,user,rss,vsz,pcpu,pmem,comm --sort=-rss | head -n 11
|
|
EOF
|
|
cmdsh "top 10 CPU hogs" <<'EOF'
|
|
ps -eo pid,user,pcpu,pmem,rss,comm --sort=-pcpu | head -n 11
|
|
EOF
|
|
cmdsh "top file-handle holders" <<'EOF'
|
|
for pid in $(ps -e -o pid= | head -n 200); do
|
|
n="$(ls /proc/$pid/fd 2>/dev/null | wc -l)"
|
|
[ "$n" -gt 50 ] && printf '%6d fd=%-6s %s\n' "$pid" "$n" "$(cat /proc/$pid/comm 2>/dev/null)"
|
|
done | sort -k2 -t= -nr | head -n 15
|
|
EOF
|
|
cmdsh "largest files in /var/log (top 10)" <<'EOF'
|
|
find /var/log -type f -size +10M -exec ls -lh {} \; 2>/dev/null | sort -k5 -h -r | head -n 10
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# 30. QUICK-LOOK SUSPECT SUMMARY (auto-detect common issues)
|
|
# ===========================================================================
|
|
hdr "30. QUICK-LOOK SUSPECT SUMMARY"
|
|
blank
|
|
printf 'Auto-detected performance anti-patterns (verify against full data above):\n' >> "$OUT"
|
|
blank
|
|
|
|
cmdsh "[A] Network interfaces < 10 Gbps and up" <<'EOF'
|
|
hits=0
|
|
for ifc in $NETIFS; do
|
|
s=$(cat "/sys/class/net/$ifc/speed" 2>/dev/null)
|
|
c=$(cat "/sys/class/net/$ifc/carrier" 2>/dev/null)
|
|
case "$s" in
|
|
10|100|1000)
|
|
if [ "$c" = "1" ]; then
|
|
printf ' %s : %s Mbps (UP) <-- sub-10GbE; suspect if this carries storage/migration\n' "$ifc" "$s"
|
|
hits=$((hits+1))
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
[ "$hits" = "0" ] && echo " no <10GbE interfaces are UP (or speed unreadable)"
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[B] MTU not jumbo on carrier-up interfaces" <<'EOF'
|
|
hits=0
|
|
for ifc in $NETIFS; do
|
|
m=$(cat "/sys/class/net/$ifc/mtu" 2>/dev/null)
|
|
c=$(cat "/sys/class/net/$ifc/carrier" 2>/dev/null)
|
|
[ "$c" = "1" ] || continue
|
|
if [ -n "$m" ] && [ "$m" != "9000" ]; then
|
|
printf ' %s : mtu=%s\n' "$ifc" "$m"
|
|
hits=$((hits+1))
|
|
fi
|
|
done
|
|
[ "$hits" = "0" ] && echo " all carrier-up interfaces have MTU 9000 (or none are up)"
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[C] CPU governor != performance" <<'EOF'
|
|
g=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null)
|
|
if [ -z "$g" ]; then
|
|
echo " cpufreq driver not loaded (governor unknown)"
|
|
elif [ "$g" = "performance" ]; then
|
|
echo " OK: performance governor"
|
|
else
|
|
echo " <-- governor=$g (recommend 'performance')"
|
|
fi
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[D] EPB / EPP not performance-bias" <<'EOF'
|
|
epb=$(cat /sys/devices/system/cpu/cpu0/power/energy_perf_bias 2>/dev/null)
|
|
epp=$(cat /sys/devices/system/cpu/cpu0/cpufreq/energy_performance_preference 2>/dev/null)
|
|
echo " EPB=$epb EPP=$epp (0=performance, 15=max-power on EPB; 'performance' on EPP)"
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[E] vm.swappiness and dirty ratios" <<'EOF'
|
|
sw=$(sysctl -n vm.swappiness 2>/dev/null)
|
|
dr=$(sysctl -n vm.dirty_ratio 2>/dev/null)
|
|
dbr=$(sysctl -n vm.dirty_background_ratio 2>/dev/null)
|
|
echo " swappiness=$sw dirty_ratio=$dr dirty_background_ratio=$dbr"
|
|
[ "$sw" != "10" ] && [ -n "$sw" ] && echo " <-- swappiness not 10 (Proxmox default)"
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[F] TCP congestion control" <<'EOF'
|
|
cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null)
|
|
avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null)
|
|
echo " active: $cc"
|
|
echo " available: $avail"
|
|
[ "$cc" = "cubic" ] && echo " <-- consider bbr (if available) for high-BDP NFS paths"
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[G] ZFS dataset red flags" <<'EOF'
|
|
if ! command -v zfs >/dev/null 2>&1; then
|
|
echo " ZFS not present"
|
|
else
|
|
zfs get -r -o name,property,value compression,atime,sync,recordsize,xattr 2>/dev/null \
|
|
| awk '$2=="compression" && $3=="off" {print " <-- "$1" has compression=off"}
|
|
$2=="atime" && $3=="on" {print " <-- "$1" has atime=on"}
|
|
$2=="sync" && $3=="always" {print " <-- "$1" has sync=always (slow without SLOG)"}
|
|
$2=="xattr" && $3=="on" {print " <-- "$1" has xattr=on (use sa)"}'
|
|
fi
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[H] ARC size vs RAM" <<'EOF'
|
|
if [ -r /proc/spl/kstat/zfs/arcstats ]; then
|
|
arc=$(awk '/^size /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
|
|
cmax=$(awk '/^c_max /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
|
|
ram=$(awk '/MemTotal/{print $2*1024}' /proc/meminfo)
|
|
echo " ARC current=$arc bytes ARC max=$cmax bytes RAM total=$ram bytes"
|
|
if [ -n "$cmax" ] && [ -n "$ram" ] && [ "$ram" -gt 0 ]; then
|
|
pct=$((cmax * 100 / ram))
|
|
echo " ARC max is ${pct}% of RAM (Proxmox default ~50%)"
|
|
fi
|
|
hits=$(awk '/^hits /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
|
|
miss=$(awk '/^misses /{print $3}' /proc/spl/kstat/zfs/arcstats 2>/dev/null)
|
|
if [ -n "$hits" ] && [ -n "$miss" ] && [ "$((hits+miss))" -gt 0 ]; then
|
|
hr=$((hits * 100 / (hits + miss)))
|
|
echo " ARC hit rate since boot: ${hr}% (hits=$hits misses=$miss)"
|
|
fi
|
|
else
|
|
echo " ZFS ARC stats not available (ZFS not loaded?)"
|
|
fi
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[I] VMs with non-optimal storage/NIC models" <<'EOF'
|
|
hits=0
|
|
for vmid in $(qm list 2>/dev/null | awk 'NR>1{print $1}'); do
|
|
cfg=$(qm config "$vmid" 2>/dev/null)
|
|
flags=""
|
|
echo "$cfg" | grep -qE '^ide[0-9]' && flags="${flags}ide-disk "
|
|
echo "$cfg" | grep -qE '^sata[0-9]' && flags="${flags}sata-disk "
|
|
echo "$cfg" | grep -qE '^scsi[0-9].*:,' && echo "$cfg" | grep -E '^scsi[0-9]' | grep -qv 'virtio-scsi' && flags="${flags}scsi-no-virtio-scsi "
|
|
echo "$cfg" | grep -qE '^scsihw: ' && ! echo "$cfg" | grep -qE '^scsihw: virtio-scsi-single' && flags="${flags}scsihw!=virtio-scsi-single "
|
|
echo "$cfg" | grep -qE '^cpu: .*kvm64' && flags="${flags}cpu=kvm64 "
|
|
echo "$cfg" | grep -q '^iothread: 1' || echo "$cfg" | grep -qE '^scsi[0-9].*iothread=1' || flags="${flags}no-iothread "
|
|
echo "$cfg" | grep -qE '^net[0-9].*model=(e1000|rtl8139|vmxnet)' && flags="${flags}legacy-nic "
|
|
echo "$cfg" | grep -qE '^net[0-9].*model=virtio' && ! echo "$cfg" | grep -qE 'multiqueue=|queues=' && flags="${flags}virtio-no-multiqueue "
|
|
if [ -n "$flags" ]; then
|
|
echo " VM $vmid: $flags"
|
|
hits=$((hits+1))
|
|
fi
|
|
done
|
|
[ "$hits" = "0" ] && echo " no obvious VM storage/NIC anti-patterns"
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[J] NFS client mount option red flags" <<'EOF'
|
|
if ! command -v nfsstat >/dev/null 2>&1; then
|
|
echo " nfsstat not available"
|
|
else
|
|
nfsstat -m 2>/dev/null | awk '
|
|
/^.*:/ {mp=$0; next}
|
|
/Flags:/ {
|
|
gsub(/Flags:|,/,"",$0)
|
|
line=$0
|
|
if (line !~ /noatime/) print " "mp" : missing noatime"
|
|
if (line !~ /nconnect/) print " "mp" : missing nconnect (kernel >=5.3)"
|
|
if (line !~ /rsize=1048576/) print " "mp" : rsize<1M"
|
|
if (line !~ /wsize=1048576/) print " "mp" : wsize<1M"
|
|
if (line ~ /sync/) print " "mp" : sync mount (slow)"
|
|
}
|
|
'
|
|
fi
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[K] Softnet / NIC drops (non-zero)" <<'EOF'
|
|
awk 'NR<=32 && ($2+0 > 0 || $4+0 > 0) {printf " cpu%d: dropped=%d time_squeezed=%d\n", NR-1, $2, $4}' /proc/net/softnet_stat
|
|
echo " (also check ethtool -S sections above for rx_dropped/rx_missed_errors)"
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[L] Failed systemd services" <<'EOF'
|
|
failed=$(systemctl --failed --no-legend 2>/dev/null | head -n 10)
|
|
if [ -n "$failed" ]; then
|
|
echo "$failed" | sed 's/^/ /'
|
|
else
|
|
echo " no failed services"
|
|
fi
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[M] Filesystem usage > 80%" <<'EOF'
|
|
df -h 2>/dev/null | awk 'NR>1 && $5+0 > 80 {print " "$0}'
|
|
EOF
|
|
|
|
blank
|
|
cmdsh "[N] SMART / NVMe health warnings" <<'EOF'
|
|
if command -v nvme >/dev/null 2>&1; then
|
|
for n in /dev/nvme[0-9]*; do
|
|
[ -b "$n" ] || continue
|
|
crit=$(nvme smart-log "$n" 2>/dev/null | grep -iE 'critical_warning|media_and_data_integrity|available_spare' | head -n 5)
|
|
[ -n "$crit" ] && echo " $n: $crit" | tr '\n' ' '
|
|
done
|
|
fi
|
|
if command -v smartctl >/dev/null 2>&1; then
|
|
for d in $(ls /sys/block/ 2>/dev/null | grep -E '^sd'); do
|
|
dev="/dev/$d"
|
|
[ -b "$dev" ] || continue
|
|
chk=$(smartctl -H "$dev" 2>/dev/null | grep -iE 'result|health|passed|failed')
|
|
echo "$chk" | grep -qi fail && echo " $dev: $chk"
|
|
done
|
|
fi
|
|
echo " (see section 20 for full smartctl/nvme output)"
|
|
EOF
|
|
|
|
# ===========================================================================
|
|
# Footer
|
|
# ===========================================================================
|
|
END_EPOCH="$(date +%s)"
|
|
DUR=$((END_EPOCH - START_EPOCH))
|
|
BYTES="$(wc -c < "$OUT")"
|
|
LINES="$(wc -l < "$OUT")"
|
|
{
|
|
printf '\n###############################################################################\n'
|
|
printf '# Collection complete: %s\n' "$HOST"
|
|
printf '# Output: %s\n' "$OUT"
|
|
printf '# Size: %s bytes / %s lines\n' "$BYTES" "$LINES"
|
|
printf '# Elapsed: %s seconds\n' "$DUR"
|
|
printf '###############################################################################\n'
|
|
} >> "$OUT"
|
|
|
|
chmod 600 "$OUT"
|
|
echo
|
|
echo "================================================================"
|
|
echo " check.sh done. Wrote: ${OUT}"
|
|
echo " Size: $(du -h "$OUT" | cut -f1) (${LINES} lines)"
|
|
echo " Elapsed: ${DUR}s"
|
|
echo " Please copy ${OUT} back for analysis."
|
|
echo "================================================================"
|