#!/bin/bash
###############################################################################
# apply-tunings.sh
#
# Applies the Tier 0 host-side tunings identified in FINDINGS.md:
#   1. Sets scaling_governor=performance on every CPU
#   2. Sets vm.swappiness appropriately (1 on storage hosts, 10 on VM hosts)
#   3. Enables tcp_bbr module + sets congestion control
#   4. Sets net.core.rmem_max/wmem_max + tcp_rmem/wmem for high-BDP NFS
#   5. Sets the recommended tuned-adm profile (network-throughput or virtual-host)
#   6. Adds nconnect=4 + noatime to NFS client mounts (edits /etc/pve/storage.cfg)
#   7. (does NOT touch bond0 xmit-hash — that requires ifreload/network
#       restart which drops the host. Provided as a separate --emit-bond-patch
#       flag that PRINTS the change but does not apply it.)
#
# Safety features:
#   - Dry-run mode by default (--apply to commit)
#   - Full backup of every modified file to /root/perfopt-backup-<timestamp>/
#   - Per-host behaviour: detects storage hosts by hostname and applies the
#     right profile (tsys4, tsys5 = storage; others = VM hosts).
#   - Generates a /root/perfopt-rollback.sh that undoes everything.
#   - Does NOT reboot, does NOT restart networking, does NOT touch hardware.
#
# Usage:
#   bash apply-tunings.sh             # dry run, show what would change
#   bash apply-tunings.sh --apply     # commit changes
#   bash apply-tunings.sh --rollback  # restore from latest backup
#   bash apply-tunings.sh --emit-bond-patch   # show bond0 hash change (no apply)
###############################################################################
set -u
umask 022

HOST="$(hostname -s)"
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
ROLLBACK_SCRIPT="/root/perfopt-rollback-${TS_SHORT}.sh"
ACTION="dryrun"
SKIP_NFS=0

while [ $# -gt 0 ]; do
    case "$1" in
        --apply)         ACTION="apply" ;;
        --rollback)      ACTION="rollback" ;;
        --emit-bond-patch) ACTION="bond-patch" ;;
        --no-nfs)        SKIP_NFS=1 ;;
        -h|--help)       sed -n '2,35p' "$0"; exit 0 ;;
        *) echo "unknown arg: $1" >&2; exit 2 ;;
    esac
    shift
done

# Detect host role from hostname
case "$HOST" in
    pfv-tsys4|pfv-tsys5)
        HOST_ROLE="storage"
        TUNED_PROFILE="network-throughput"
        SWAPPINESS="1"
        ;;
    pfv-tsys1|pfv-tsys3|pfv-tsys6|pfv-tsys7)
        HOST_ROLE="vmhost"
        TUNED_PROFILE="virtual-host"
        SWAPPINESS="10"
        ;;
    *)
        HOST_ROLE="unknown"
        TUNED_PROFILE="virtual-host"
        SWAPPINESS="10"
        ;;
esac

# --- rollback path ---------------------------------------------------------
if [ "$ACTION" = "rollback" ]; then
    echo "Looking for latest backup under /root/perfopt-backup-* ..."
    latest=""
    while IFS= read -r d; do
        latest="$d"
    done < <(find /root -maxdepth 1 -type d -name 'perfopt-backup-*' 2>/dev/null | sort | tail -n1)
    if [ -z "$latest" ]; then
        echo "No backup found under /root/perfopt-backup-*" >&2
        exit 1
    fi
    rb="$latest/perfopt-rollback.sh"
    if [ ! -x "$rb" ] && [ ! -r "$rb" ]; then
        echo "Rollback script missing in $latest" >&2
        exit 1
    fi
    echo "Rolling back using: $rb"
    bash "$rb"
    exit $?
fi

# --- bond patch print-only path -------------------------------------------
if [ "$ACTION" = "bond-patch" ]; then
    echo "==================================================================="
    echo " Proposed bond0 xmit_hash_policy change"
    echo "==================================================================="
    echo
    if [ ! -r /etc/network/interfaces ]; then
        echo "/etc/network/interfaces not readable"
        exit 1
    fi
    if ! grep -q 'bond-mode 802.3ad\|bond-mode 4\|bond-slaves' /etc/network/interfaces; then
        echo "No bond0 detected on this host — nothing to patch."
        exit 0
    fi
    cat <<EOF
A manual edit to /etc/network/interfaces is required. The bond0 stanza
needs this line added (it defaults to layer2, which is wrong for storage):

    bond-xmit-hash-policy layer3+4

After editing, you MUST reload networking for it to take effect:
    ifreload -a            # safe, brings interfaces down/up
    # OR
    systemctl restart networking   # heavier, briefly drops connections

WARNING: applying this on a remote host over bond0 will briefly drop your
SSH session. Run from console/IPMI, or schedule a maintenance window.

The change is reversible by removing the line and reloading again.
EOF
    exit 0
fi

# --- main path (dryrun or apply) ------------------------------------------
mkdir -p "$BACKUP_DIR"

# Emit the rollback script header
cat > "$ROLLBACK_SCRIPT" <<EOF
#!/bin/bash
# Auto-generated rollback for perfopt apply-tunings.sh
# Backup timestamp: $TS
# Backup dir:       $BACKUP_DIR
# Generated on:     $HOST
set -u
EOF
chmod +x "$ROLLBACK_SCRIPT"

backup_file() {
    local f="$1"
    if [ -e "$f" ]; then
        cp -a "$f" "$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
    fi
}

# Helper: append a restore command to the rollback script
rb_restore() {
    local f="$1"
    local bk
    bk="$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
    cat >> "$ROLLBACK_SCRIPT" <<EOF
if [ -r "$bk" ]; then
    cp -a "$bk" "$f" && echo "restored $f"
else
    echo "WARN: backup $bk missing — $f left as-is"
fi
EOF
}

# Helper: append a sysctl restore command
rb_sysctl_restore() {
    local key="$1" orig_val="$2"
    cat >> "$ROLLBACK_SCRIPT" <<EOF
sysctl -w "$key=$orig_val" >/dev/null && echo "restored $key=$orig_val"
EOF
}

# Helper: append a tuned-adm restore
rb_tuned_restore() {
    local profile="$1"
    cat >> "$ROLLBACK_SCRIPT" <<EOF
tuned-adm profile "$profile" 2>/dev/null && echo "restored tuned profile: $profile"
EOF
}

echo "==================================================================="
echo " perfopt apply-tunings — $HOST"
echo " mode: $ACTION"
echo " role: $HOST_ROLE (tuned=$TUNED_PROFILE, swappiness=$SWAPPINESS)"
echo " backup: $BACKUP_DIR"
echo " rollback: $ROLLBACK_SCRIPT"
echo "==================================================================="
echo

# ===========================================================================
# 1. scaling_governor → performance
# ===========================================================================
echo "--- 1. CPU scaling_governor → performance"
if [ -r /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
    cur=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
    echo "    current: $cur"
    if [ "$ACTION" = "apply" ]; then
        if [ "$cur" != "performance" ]; then
            for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
                echo performance > "$c" 2>/dev/null || true
            done
            # Persist via systemd tmpfiles / sysctl fallback
            cat > /etc/systemd/system/perfopt-cpu-performance.service <<EOF
[Unit]
Description=Set CPU scaling_governor=performance (perfopt)
After=multi-user.target

[Service]
Type=oneshot
ExecStart=/bin/sh -c 'for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "\$c" 2>/dev/null || true; done'
RemainAfterExit=yes

[Install]
WantedBy=multi-user.target
EOF
            systemctl daemon-reload
            systemctl enable perfopt-cpu-performance.service 2>/dev/null
            systemctl start perfopt-cpu-performance.service 2>/dev/null
            echo "    applied + persisted via systemd unit"
            # Rollback
            cat >> "$ROLLBACK_SCRIPT" <<EOF
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "$cur" > "\$c" 2>/dev/null || true; done
systemctl disable --now perfopt-cpu-performance.service 2>/dev/null || true
rm -f /etc/systemd/system/perfopt-cpu-performance.service
systemctl daemon-reload
echo "restored scaling_governor=$cur (best-effort; original may have been dynamic)"
EOF
        else
            echo "    already performance — no change"
        fi
    fi
else
    echo "    cpufreq driver not loaded — nothing to do (typical on BIOS-locked servers)"
fi

# ===========================================================================
# 2. vm.swappiness
# ===========================================================================
echo "--- 2. vm.swappiness → $SWAPPINESS (role=$HOST_ROLE)"
cur_swappiness=$(sysctl -n vm.swappiness 2>/dev/null || echo "?")
echo "    current: $cur_swappiness"
if [ "$ACTION" = "apply" ] && [ "$cur_swappiness" != "$SWAPPINESS" ]; then
    # Persist via sysctl.d
    backup_file /etc/sysctl.d/99-perfopt.conf
    cat > /etc/sysctl.d/99-perfopt.conf <<EOF
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
vm.swappiness = $SWAPPINESS
EOF
    sysctl -w "vm.swappiness=$SWAPPINESS" >/dev/null
    echo "    applied (persisted to /etc/sysctl.d/99-perfopt.conf)"
    rb_restore /etc/sysctl.d/99-perfopt.conf
    cat >> "$ROLLBACK_SCRIPT" <<EOF
rm -f /etc/sysctl.d/99-perfopt.conf
sysctl -w "vm.swappiness=$cur_swappiness" >/dev/null
echo "restored vm.swappiness=$cur_swappiness"
EOF
else
    echo "    no change"
fi

# ===========================================================================
# 3. TCP BBR + buffers  (single sysctl.d file)
# ===========================================================================
echo "--- 3. TCP BBR + socket buffers"
cur_cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo "?")
cur_avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null || echo "?")
echo "    current: $cur_cc (available: $cur_avail)"

# Decide sysctl values (these are the consensus values for a 1-10 GbE NFS host)
TARGET_RMEM_MAX=134217728        # 128 MB
TARGET_WMEM_MAX=134217728
TARGET_RMEM_DEFAULT=26214400     # 25 MB
TARGET_WMEM_DEFAULT=26214400
TARGET_TCP_RMEM="4096 87380 $TARGET_RMEM_MAX"
TARGET_TCP_WMEM="4096 65536 $TARGET_RMEM_MAX"
TARGET_NETDEV_MAX_BACKLOG=250000
TARGET_SOMAXCONN=65535

if [ "$ACTION" = "apply" ]; then
    # Ensure tcp_bbr module loads at boot
    if ! grep -q '^tcp_bbr' /etc/modules-load.d/modules.conf 2>/dev/null; then
        backup_file /etc/modules-load.d/modules.conf
        mkdir -p /etc/modules-load.d
        cat >> /etc/modules-load.d/modules.conf <<EOF
# perfopt apply-tunings.sh — $TS
tcp_bbr
EOF
        modprobe tcp_bbr 2>/dev/null || true
        rb_restore /etc/modules-load.d/modules.conf
    fi

    backup_file /etc/sysctl.d/99-perfopt.conf
    cat > /etc/sysctl.d/99-perfopt.conf <<EOF
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
vm.swappiness = $SWAPPINESS

# TCP BBR + high-BDP buffers (good for NFS over 1-10 GbE)
net.ipv4.tcp_congestion_control = bbr
net.core.default_qdisc = fq
net.core.rmem_max = $TARGET_RMEM_MAX
net.core.wmem_max = $TARGET_WMEM_MAX
net.core.rmem_default = $TARGET_RMEM_DEFAULT
net.core.wmem_default = $TARGET_WMEM_DEFAULT
net.core.netdev_max_backlog = $TARGET_NETDEV_MAX_BACKLOG
net.core.somaxconn = $TARGET_SOMAXCONN
net.ipv4.tcp_rmem = $TARGET_TCP_RMEM
net.ipv4.tcp_wmem = $TARGET_TCP_WMEM
EOF
    sysctl --system >/dev/null 2>&1
    echo "    applied (written to /etc/sysctl.d/99-perfopt.conf + sysctl --system)"
    # Rebuild rollback for sysctl
    cat >> "$ROLLBACK_SCRIPT" <<EOF
# Restore original sysctl state
rm -f /etc/sysctl.d/99-perfopt.conf
EOF
    rb_sysctl_restore "net.ipv4.tcp_congestion_control" "$cur_cc"
    rb_sysctl_restore "vm.swappiness" "$cur_swappiness"
    # Note: original buffer sizes not all captured; restoration is best-effort.
    cat >> "$ROLLBACK_SCRIPT" <<EOF
echo "Note: net buffers restored to kernel defaults (original values not captured)."
echo "Run 'sysctl --system' to apply."
sysctl --system >/dev/null 2>&1 || true
EOF
fi

# ===========================================================================
# 4. tuned-adm profile
# ===========================================================================
echo "--- 4. tuned-adm profile → $TUNED_PROFILE"
if command -v tuned-adm >/dev/null 2>&1; then
    cur_profile=$(tuned-adm active 2>/dev/null | awk -F: '/Current active profile/{gsub(/^[ \t]+/,"",$2); print $2}')
    echo "    current: $cur_profile"
    if [ "$ACTION" = "apply" ] && [ "$cur_profile" != "$TUNED_PROFILE" ]; then
        tuned-adm profile "$TUNED_PROFILE" 2>&1 | sed 's/^/    /'
        rb_tuned_restore "$cur_profile"
    else
        echo "    no change"
    fi
else
    echo "    tuned-adm not installed — skipping"
fi

# ===========================================================================
# 5. NFS mount option hardening
#    Edit /etc/pve/storage.cfg to add nconnect=4 + noatime to NFS plugins.
#    Proxmox applies these on next mount/remount.
#    Skipped with --no-nfs (for NFS servers where we don't want to remount).
# ===========================================================================
echo "--- 5. NFS mount options: add nconnect=4, noatime"
if [ "$SKIP_NFS" -eq 1 ]; then
    echo "    SKIPPED (--no-nfs specified)"
    echo "    Note: storage.cfg is cluster-wide; if edited on another host,"
    echo "    the options are already staged here and activate on next reboot."
else
if [ -r /etc/pve/storage.cfg ]; then
    backup_file /etc/pve/storage.cfg
    if grep -q '^nfs:' /etc/pve/storage.cfg; then
        # Count how many nfs: stanzas lack the options
        while IFS= read -r line; do
            if echo "$line" | grep -q '^nfs:'; then
                cur_stanza=$(echo "$line" | awk '{print $2}')
                # Look ahead for the next few lines to see if options already set
                echo "    nfs stanza: $cur_stanza"
            fi
        done < /etc/pve/storage.cfg

        # Check whether any nfs stanza already has options
        if grep -A1 '^nfs:' /etc/pve/storage.cfg | grep -q 'options.*nconnect=4'; then
            echo "    some stanzas already have nconnect=4 — check manually"
        else
            echo "    proposed change: add 'options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2' to each nfs stanza"
            if [ "$ACTION" = "apply" ]; then
                # Apply via perl for safety (in-place edit with backup)
                if cp -a /etc/pve/storage.cfg "$BACKUP_DIR/etc__pve__storage.cfg"; then
                    # Use a python helper for robust PVE storage.cfg editing
                    python3 - <<PYEOF
import re, pathlib
p = pathlib.Path("/etc/pve/storage.cfg")
text = p.read_text()
pattern = re.compile(r'(^nfs:\s*\S+\n(?:[ \t]+[^\n]+\n)+)', re.MULTILINE)
modified = 0
def fix(m):
    global modified
    block = m.group(1)
    if 'options' in block:
        return block
    lines = block.rstrip('\n').split('\n')
    lines.insert(1, '\toptions nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2')
    modified += 1
    return '\n'.join(lines) + '\n'
new_text = pattern.sub(fix, text)
if modified:
    p.write_text(new_text)
    print(f"    patched {modified} NFS stanzas with options line")
else:
    print("    no NFS stanza needed patching (all already had options or no nfs: found)")
PYEOF
                    # Remount existing NFS mounts to pick up new options
                    echo "    remounting NFS mounts to apply new options..."
                    while IFS= read -r mp; do
                        mount -o remount "$mp" 2>/dev/null && echo "      remounted $mp" || echo "      FAILED to remount $mp (will pick up on next mount)"
                    done < <(awk '$3 ~ /^nfs/{print $2}' /proc/mounts 2>/dev/null | sort -u)
                    rb_restore /etc/pve/storage.cfg
                    cat >> "$ROLLBACK_SCRIPT" <<EOF
# To fully undo NFS options, also remount:
while IFS= read -r mp; do
    mount -o remount "\$mp" 2>/dev/null
done < <(awk '\$3 ~ /^nfs/{print \$2}' /proc/mounts 2>/dev/null | sort -u)
echo "restored NFS mount options"
EOF
                else
                    echo "    ERROR: could not back up storage.cfg — aborting NFS patch"
                fi
            fi
        fi
    else
        echo "    no NFS stanzas in storage.cfg — skipping"
    fi
else
    echo "    /etc/pve/storage.cfg not readable — skipping"
fi
fi

# ===========================================================================
# 6. Summary
# ===========================================================================
echo
echo "==================================================================="
if [ "$ACTION" = "apply" ]; then
    echo " DONE. Backups in: $BACKUP_DIR"
    echo " Rollback: bash $ROLLBACK_SCRIPT"
    echo
    echo " Next steps:"
    echo "   - Verify with 'sysctl net.ipv4.tcp_congestion_control vm.swappiness'"
    echo "   - Verify NFS mounts with 'nfsstat -m' (look for nconnect=4)"
    echo "   - Run 'bash apply-tunings.sh --emit-bond-patch' for the bond0"
    echo "     xmit_hash_policy change (requires network restart, do in"
    echo "     maintenance window)"
else
    echo " DRY RUN — no changes made."
    echo " To commit: bash apply-tunings.sh --apply"
fi
echo "==================================================================="