PFVCluster/validate-fixes.sh

#!/bin/bash
# validate-fixes.sh - READ-ONLY validation of all applied tunings.
# Does NOT reboot, shutdown VMs, or modify anything.
set -uo pipefail
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)

HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys6 pfv-tsys7 pfv-tsys9)

echo "==================================================================="
echo " READ-ONLY VALIDATION — $(date)"
echo "==================================================================="
echo ""

for HOST in "${HOSTS[@]}"; do
    echo "================================================================"
    echo "[$HOST]"
    echo "================================================================"

    if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
        echo "  UNREACHABLE"
        echo ""
        continue
    fi

    ssh "${SSH_OPTS[@]}" "root@$HOST" '
        pass=0; fail=0
        check() {
            local label="$1" actual="$2" expected="$3"
            if [ "$actual" = "$expected" ]; then
                printf "  [OK]   %-30s %s\n" "$label" "$actual"
                pass=$((pass+1))
            else
                printf "  [FAIL] %-30s got=%s want=%s\n" "$label" "$actual" "$expected"
                fail=$((fail+1))
            fi
        }

        # 1. CPU governor
        gov=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "n/a")
        if [ "$gov" = "n/a" ]; then
            printf "  [SKIP] %-30s %s\n" "CPU governor" "(no cpufreq driver — OK for server BIOS)"
        else
            check "CPU governor" "$gov" "performance"
        fi

        # 2. vm.swappiness
        swap=$(sysctl -n vm.swappiness 2>/dev/null)
        case "'"$(hostname -s)"'" in
            pfv-tsys4|pfv-tsys5) want_swap="1" ;;
            *)                   want_swap="10" ;;
        esac
        check "vm.swappiness" "$swap" "$want_swap"

        # 3. TCP congestion control
        cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null)
        check "tcp_congestion_control" "$cc" "bbr"

        # 4. default_qdisc (paired with BBR)
        qd=$(sysctl -n net.core.default_qdisc 2>/dev/null)
        check "net.core.default_qdisc" "$qd" "fq"

        # 5. tuned profile
        if command -v tuned-adm >/dev/null 2>&1; then
            tuned=$(tuned-adm active 2>/dev/null | awk -F: "/Current active/{gsub(/^[ \t]+/,\"\",\$2); print \$2}")
            case "'"$(hostname -s)"'" in
                pfv-tsys4|pfv-tsys5) want_tuned="throughput-performance" ;;
                *)                   want_tuned="virtual-host" ;;
            esac
            check "tuned-adm profile" "$tuned" "$want_tuned"
        else
            printf "  [FAIL] %-30s not installed\n" "tuned-adm"
            fail=$((fail+1))
        fi

        # 6. bond0 hash policy (if bond exists)
        if [ -r /proc/net/bonding/bond0 ]; then
            hash=$(grep "Transmit Hash" /proc/net/bonding/bond0 2>/dev/null | awk "{print \$4}")
            check "bond0 xmit_hash_policy" "$hash" "layer3+4"
            ports=$(grep "Number of ports" /proc/net/bonding/bond0 2>/dev/null | awk "{print \$4}")
            printf "  [INFO] %-30s %s ports active\n" "bond0 LACP ports" "$ports"
        else
            printf "  [SKIP] %-30s %s\n" "bond0 hash" "(no bond0 — single NIC host)"
        fi

        # 7. NFS mount options (nconnect + noatime)
        nfs_first=$(nfsstat -m 2>/dev/null | head -3 | tail -1)
        if echo "$nfs_first" | grep -q "nconnect=4"; then
            printf "  [OK]   %-30s nconnect=4 active\n" "NFS nconnect"
            pass=$((pass+1))
        elif echo "$nfs_first" | grep -q "relatime"; then
            printf "  [FAIL] %-30s still relatime (needs reboot/mount)\n" "NFS nconnect"
            fail=$((fail+1))
        elif [ -z "$nfs_first" ]; then
            printf "  [WARN] %-30s no NFS mounts (lazy — start a VM)\n" "NFS nconnect"
        else
            printf "  [FAIL] %-30s unexpected: %s\n" "NFS nconnect" "$nfs_first"
            fail=$((fail+1))
        fi

        if echo "$nfs_first" | grep -q "noatime"; then
            printf "  [OK]   %-30s noatime active\n" "NFS noatime"
            pass=$((pass+1))
        elif [ -n "$nfs_first" ]; then
            printf "  [FAIL] %-30s not noatime\n" "NFS noatime"
            fail=$((fail+1))
        fi

        # 8. NFS TCP connection count
        nfs_conns=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
        if [ "$nfs_conns" -ge 8 ]; then
            printf "  [OK]   %-30s %s connections\n" "NFS TCP conns" "$nfs_conns"
            pass=$((pass+1))
        elif [ "$nfs_conns" -gt 0 ]; then
            printf "  [WARN] %-30s %s (expect 8 with nconnect=4)\n" "NFS TCP conns" "$nfs_conns"
        else
            printf "  [WARN] %-30s 0 (lazy mounts — start a VM)\n" "NFS TCP conns"
        fi

        # 9. sysctl persistence
        if [ -r /etc/sysctl.d/99-perfopt.conf ]; then
            printf "  [OK]   %-30s /etc/sysctl.d/99-perfopt.conf\n" "sysctl persistence"
            pass=$((pass+1))
        else
            printf "  [FAIL] %-30s missing\n" "sysctl persistence"
            fail=$((fail+1))
        fi

        # 10. Observability packages
        for cmd in sar jq numactl nvme mtr bmon; do
            if ! command -v "$cmd" >/dev/null 2>&1; then
                printf "  [FAIL] %-30s not installed\n" "obs: $cmd"
                fail=$((fail+1))
            fi
        done
        if command -v sar >/dev/null 2>&1 && command -v jq >/dev/null 2>&1 && \
           command -v numactl >/dev/null 2>&1 && command -v nvme >/dev/null 2>&1 && \
           command -v mtr >/dev/null 2>&1 && command -v bmon >/dev/null 2>&1; then
            printf "  [OK]   %-30s all installed\n" "observability packages"
            pass=$((pass+1))
        fi

        # 11. Failed services
        failed_count=$(systemctl --failed --no-legend 2>/dev/null | wc -l)
        if [ "$failed_count" = "0" ]; then
            printf "  [OK]   %-30s none\n" "failed services"
            pass=$((pass+1))
        else
            printf "  [FAIL] %-30s %s failed:\n" "failed services" "$failed_count"
            systemctl --failed --no-legend 2>/dev/null | sed "s/^/         /"
            fail=$((fail+1))
        fi

        # 12. VM status (read-only — just report)
        running_vms=$(qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"" | wc -l)
        stopped_vms=$(qm list 2>/dev/null | awk "NR>1 && \$3!=\"running\"" | wc -l)
        printf "  [INFO] %-30s %s running, %s stopped\n" "VM status" "$running_vms" "$stopped_vms"

        # 13. uptime
        printf "  [INFO] %-30s %s\n" "uptime" "$(uptime | sed "s/.*up //" | sed "s/,.*//")"

        echo ""
        echo "  RESULT: $pass passed, $fail failed"
    '
    echo ""
done

echo "==================================================================="
echo " SUMMARY"
echo "==================================================================="