PFVCluster/proxmox/perf/scripts/fix-tsys4-storage-bond.sh

#!/bin/bash
###############################################################################
# fix-tsys4-storage-bond.sh
#
# Reconfigures pfv-tsys4 storage network bond from broken 802.3ad (LACP) to
# active-backup mode. USB NICs cannot participate in LACP — the driver does
# not report speed/duplex to the kernel (ethtool shows "Speed: Unknown!"),
# so the bonding driver never transmits LACP PDUs. active-backup requires no
# switch-side LACP and works correctly with a single port.
#
# Run ON pfv-tsys4. Designed for maintenance-window execution.
#
# Safety:
#   - Dry-run by default (--apply to commit)
#   - Full backup of /etc/network/interfaces
#   - 5 health checks with automatic rollback on failure
#   - SSH survives (management on vmbr0/tailscale0, not bond0/datanet)
#   - Pre-generated rollback script for manual recovery
#
# Switch side (core-sw01): NO changes needed. g31 is already a standalone
# access port in VLAN 1000 — correct for active-backup (no LACP required).
# g32/ch2 cleanup (dead NIC) is left for separate maintenance.
#
# Usage:
#   bash fix-tsys4-storage-bond.sh             # dry-run (show changes only)
#   bash fix-tsys4-storage-bond.sh --apply     # commit with auto-rollback
#
# Related: Redmine [#394] BUG 4
###############################################################################
set -euo pipefail

STORAGE_PEER="10.100.100.6"
INTERFACES="/etc/network/interfaces"
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
BACKUP_DIR="/root/tsys4-bondfix-backup-${TS_SHORT}"
ROLLBACK="/root/tsys4-bondfix-rollback-${TS_SHORT}.sh"
ACTION="${1:-dryrun}"

[ "${ACTION}" = "--apply" ] && ACTION="apply" || ACTION="dryrun"

echo "==================================================================="
echo " fix-tsys4-storage-bond — $(hostname -s)"
echo " mode: ${ACTION}"
echo " time: $(date)"
echo "==================================================================="
echo ""

# -------------------------------------------------------------------------
# Pre-flight checks
# -------------------------------------------------------------------------
echo "--- Pre-flight checks ---"

# Must be pfv-tsys4
HOSTNAME_S="$(hostname -s)"
if [ "${HOSTNAME_S}" != "pfv-tsys4" ]; then
    echo "FATAL: This script targets pfv-tsys4 (this host: ${HOSTNAME_S})"
    exit 1
fi
echo "  Host: OK (pfv-tsys4)"

# bond0 must exist
if [ ! -d /sys/class/net/bond0 ]; then
    echo "FATAL: bond0 not found — no bond to fix"
    exit 1
fi
echo "  bond0: present"

# ifreload must be available
if ! command -v ifreload >/dev/null 2>&1; then
    echo "FATAL: ifreload not found (need ifupdown2)"
    exit 1
fi
echo "  ifreload: available"

# SSH must NOT be on bond0/datanet (check incoming route)
SSH_SRC="$(echo "${SSH_CLIENT:-}" | awk '{print $1}')"
if [ -n "${SSH_SRC}" ]; then
    SSH_IFACE="$(ip route get "${SSH_SRC}" 2>/dev/null | grep -oP 'dev \K\S+' || echo "unknown")"
    echo "  SSH ingress: ${SSH_IFACE}"
    if echo "${SSH_IFACE}" | grep -qE 'bond0|datanet'; then
        echo "FATAL: SSH is on storage network — cannot safely reload."
        echo "       Use physical console (pfv-tsys4) to run this script."
        exit 1
    fi
else
    echo "  SSH ingress: (local/console session — OK)"
fi
echo "  SSH safety: OK (not on storage network)"
echo ""

# -------------------------------------------------------------------------
# Show current state
# -------------------------------------------------------------------------
echo "--- Current bond0 state ---"
grep -E "Bonding Mode|Transmit Hash|MII Status|Number of ports" /proc/net/bonding/bond0
echo ""
grep -E "Slave Interface|MII Status|Speed" /proc/net/bonding/bond0
echo ""

echo "--- Current bond0 stanza in /etc/network/interfaces ---"
awk '/^auto bond0/,/^$/' "${INTERFACES}"
echo ""

# -------------------------------------------------------------------------
# Baseline connectivity
# -------------------------------------------------------------------------
echo "--- Baseline connectivity ---"
echo -n "  Ping ${STORAGE_PEER}: "
if ping -c 1 -W 2 "${STORAGE_PEER}" >/dev/null 2>&1; then
    echo "OK"
else
    echo "UNREACHABLE (baseline already broken — proceed with caution)"
fi
echo -n "  NFS server: "
systemctl is-active nfs-server 2>/dev/null || echo "(not active)"
echo ""

# -------------------------------------------------------------------------
# Backup
# -------------------------------------------------------------------------
mkdir -p "${BACKUP_DIR}"
cp -a "${INTERFACES}" "${BACKUP_DIR}/interfaces"
echo "Backup: ${BACKUP_DIR}/interfaces"

# Generate rollback script
cat > "${ROLLBACK}" <<ROLLBACKEOF
#!/bin/bash
set -euo pipefail
echo "Rolling back tsys4 bond config..."
cp -a "${BACKUP_DIR}/interfaces" "${INTERFACES}"
echo "Restored ${INTERFACES}"
ifreload -a 2>&1 || systemctl restart networking 2>&1 || true
sleep 3
echo "Post-rollback bond0 state:"
grep -E "Bonding Mode|MII Status|Slave Interface|Speed" /proc/net/bonding/bond0 2>/dev/null
echo "Rollback complete."
ROLLBACKEOF
chmod +x "${ROLLBACK}"
echo "Rollback: ${ROLLBACK}"
echo ""

# -------------------------------------------------------------------------
# Describe the change
# -------------------------------------------------------------------------
echo "==================================================================="
echo " PROPOSED CHANGE"
echo "==================================================================="
echo "  1. bond-mode:  802.3ad → active-backup"
echo "  2. Remove bond-xmit-hash-policy (unused in active-backup)"
echo "  3. bond-slaves: keep enx8cae4ccda926 (active NIC only)"
echo ""
echo "  Rationale: USB NICs cannot do LACP. ethtool reports Speed: Unknown,"
echo "  so the bonding driver never sends LACP PDUs (verified via tcpdump:"
echo "  0 LACP PDUs in 65s on both slave and bond master). active-backup"
echo "  needs no LACP and works with the single working port."
echo ""
echo "  Switch: NO changes needed. g31 is standalone access VLAN 1000."
echo ""

if [ "${ACTION}" != "apply" ]; then
    echo "==================================================================="
    echo " DRY RUN — no changes made."
    echo " Commit:  bash \$0 --apply"
    echo "==================================================================="
    exit 0
fi

# =========================================================================
# APPLY
# =========================================================================
echo "==================================================================="
echo " APPLYING — auto-rollback on health check failure"
echo "==================================================================="
echo ""

# 1. Change bond-mode
echo "Changing bond-mode 802.3ad → active-backup..."
sed -i 's/bond-mode 802\.3ad/bond-mode active-backup/' "${INTERFACES}"

# 2. Remove bond-xmit-hash-policy (not used by active-backup)
echo "Removing bond-xmit-hash-policy..."
sed -i '/bond-xmit-hash-policy/d' "${INTERFACES}"

# Show updated stanza
echo ""
echo "--- Updated bond0 stanza ---"
awk '/^auto bond0/,/^$/' "${INTERFACES}"
echo ""

# 3. Reload networking
echo "--- Reloading networking (ifreload -a) ---"
echo "SSH safe on vmbr0/tailscale0 (not bond0)..."
ifreload -a 2>&1 || echo "WARNING: ifreload returned non-zero — checking state..."

# Wait for bond to settle
echo "Waiting 5s for bond to settle..."
sleep 5

# =========================================================================
# HEALTH CHECKS
# =========================================================================
echo ""
echo "==================================================================="
echo " HEALTH CHECKS (5 tests)"
echo "==================================================================="

HEALTH_OK=true

# Check 1: bond0 is UP
echo -n "  [1/5] bond0 MII up: "
if grep -q "MII Status: up" /proc/net/bonding/bond0 2>/dev/null; then
    echo "PASS"
else
    echo "FAIL"
    HEALTH_OK=false
fi

# Check 2: Active slave assigned
echo -n "  [2/5] Active slave: "
ACTIVE_SLAVE="$(cat /sys/class/net/bond0/bonding/active_slave 2>/dev/null || echo "")"
if [ -n "${ACTIVE_SLAVE}" ]; then
    echo "PASS (${ACTIVE_SLAVE})"
else
    echo "FAIL (no active slave)"
    HEALTH_OK=false
fi

# Check 3: datanet bridge UP
echo -n "  [3/5] datanet bridge up: "
if ip link show datanet 2>/dev/null | grep -q "state UP"; then
    echo "PASS"
else
    echo "FAIL"
    HEALTH_OK=false
fi

# Check 4: Storage network IP present on datanet
echo -n "  [4/5] Storage IP (10.100.100.4): "
if ip addr show datanet 2>/dev/null | grep -q "10.100.100.4"; then
    echo "PASS"
else
    echo "FAIL"
    HEALTH_OK=false
fi

# Check 5: Storage peer reachable
echo -n "  [5/5] Ping ${STORAGE_PEER}: "
if ping -c 3 -W 2 "${STORAGE_PEER}" >/dev/null 2>&1; then
    echo "PASS"
else
    echo "FAIL"
    HEALTH_OK=false
fi

# =========================================================================
# COMMIT OR ROLLBACK
# =========================================================================
echo ""

if [ "${HEALTH_OK}" = "true" ]; then
    echo "==================================================================="
    echo " SUCCESS — all 5 health checks passed"
    echo "==================================================================="
    echo ""
    echo "--- Final bond0 state ---"
    grep -E "Bonding Mode|MII Status|Slave Interface|Speed|Active" /proc/net/bonding/bond0
    echo ""
    echo "Backup:   ${BACKUP_DIR}/interfaces"
    echo "Rollback: ${ROLLBACK}"
    echo ""
    echo "Config change is LIVE but not yet reboot-tested."
    echo "Verify NFS clients are healthy before next maintenance window."
    exit 0
else
    echo "==================================================================="
    echo " HEALTH CHECK FAILED — auto-rolling back"
    echo "==================================================================="
    bash "${ROLLBACK}"
    echo ""
    echo "Auto-rollback complete."
    echo "Original config restored. Manual investigation needed."
    exit 1
fi