diff --git a/proxmox/perf/scripts/fix-tsys4-storage-bond.sh b/proxmox/perf/scripts/fix-tsys4-storage-bond.sh new file mode 100644 index 0000000..689d172 --- /dev/null +++ b/proxmox/perf/scripts/fix-tsys4-storage-bond.sh @@ -0,0 +1,283 @@ +#!/bin/bash +############################################################################### +# fix-tsys4-storage-bond.sh +# +# Reconfigures pfv-tsys4 storage network bond from broken 802.3ad (LACP) to +# active-backup mode. USB NICs cannot participate in LACP — the driver does +# not report speed/duplex to the kernel (ethtool shows "Speed: Unknown!"), +# so the bonding driver never transmits LACP PDUs. active-backup requires no +# switch-side LACP and works correctly with a single port. +# +# Run ON pfv-tsys4. Designed for maintenance-window execution. +# +# Safety: +# - Dry-run by default (--apply to commit) +# - Full backup of /etc/network/interfaces +# - 5 health checks with automatic rollback on failure +# - SSH survives (management on vmbr0/tailscale0, not bond0/datanet) +# - Pre-generated rollback script for manual recovery +# +# Switch side (core-sw01): NO changes needed. g31 is already a standalone +# access port in VLAN 1000 — correct for active-backup (no LACP required). +# g32/ch2 cleanup (dead NIC) is left for separate maintenance. +# +# Usage: +# bash fix-tsys4-storage-bond.sh # dry-run (show changes only) +# bash fix-tsys4-storage-bond.sh --apply # commit with auto-rollback +# +# Related: Redmine [#394] BUG 4 +############################################################################### +set -euo pipefail + +STORAGE_PEER="10.100.100.6" +INTERFACES="/etc/network/interfaces" +TS_SHORT="$(date +%Y%m%d-%H%M%S)" +BACKUP_DIR="/root/tsys4-bondfix-backup-${TS_SHORT}" +ROLLBACK="/root/tsys4-bondfix-rollback-${TS_SHORT}.sh" +ACTION="${1:-dryrun}" + +[ "${ACTION}" = "--apply" ] && ACTION="apply" || ACTION="dryrun" + +echo "===================================================================" +echo " fix-tsys4-storage-bond — $(hostname -s)" +echo " mode: ${ACTION}" +echo " time: $(date)" +echo "===================================================================" +echo "" + +# ------------------------------------------------------------------------- +# Pre-flight checks +# ------------------------------------------------------------------------- +echo "--- Pre-flight checks ---" + +# Must be pfv-tsys4 +HOSTNAME_S="$(hostname -s)" +if [ "${HOSTNAME_S}" != "pfv-tsys4" ]; then + echo "FATAL: This script targets pfv-tsys4 (this host: ${HOSTNAME_S})" + exit 1 +fi +echo " Host: OK (pfv-tsys4)" + +# bond0 must exist +if [ ! -d /sys/class/net/bond0 ]; then + echo "FATAL: bond0 not found — no bond to fix" + exit 1 +fi +echo " bond0: present" + +# ifreload must be available +if ! command -v ifreload >/dev/null 2>&1; then + echo "FATAL: ifreload not found (need ifupdown2)" + exit 1 +fi +echo " ifreload: available" + +# SSH must NOT be on bond0/datanet (check incoming route) +SSH_SRC="$(echo "${SSH_CLIENT:-}" | awk '{print $1}')" +if [ -n "${SSH_SRC}" ]; then + SSH_IFACE="$(ip route get "${SSH_SRC}" 2>/dev/null | grep -oP 'dev \K\S+' || echo "unknown")" + echo " SSH ingress: ${SSH_IFACE}" + if echo "${SSH_IFACE}" | grep -qE 'bond0|datanet'; then + echo "FATAL: SSH is on storage network — cannot safely reload." + echo " Use physical console (pfv-tsys4) to run this script." + exit 1 + fi +else + echo " SSH ingress: (local/console session — OK)" +fi +echo " SSH safety: OK (not on storage network)" +echo "" + +# ------------------------------------------------------------------------- +# Show current state +# ------------------------------------------------------------------------- +echo "--- Current bond0 state ---" +grep -E "Bonding Mode|Transmit Hash|MII Status|Number of ports" /proc/net/bonding/bond0 +echo "" +grep -E "Slave Interface|MII Status|Speed" /proc/net/bonding/bond0 +echo "" + +echo "--- Current bond0 stanza in /etc/network/interfaces ---" +awk '/^auto bond0/,/^$/' "${INTERFACES}" +echo "" + +# ------------------------------------------------------------------------- +# Baseline connectivity +# ------------------------------------------------------------------------- +echo "--- Baseline connectivity ---" +echo -n " Ping ${STORAGE_PEER}: " +if ping -c 1 -W 2 "${STORAGE_PEER}" >/dev/null 2>&1; then + echo "OK" +else + echo "UNREACHABLE (baseline already broken — proceed with caution)" +fi +echo -n " NFS server: " +systemctl is-active nfs-server 2>/dev/null || echo "(not active)" +echo "" + +# ------------------------------------------------------------------------- +# Backup +# ------------------------------------------------------------------------- +mkdir -p "${BACKUP_DIR}" +cp -a "${INTERFACES}" "${BACKUP_DIR}/interfaces" +echo "Backup: ${BACKUP_DIR}/interfaces" + +# Generate rollback script +cat > "${ROLLBACK}" <&1 || systemctl restart networking 2>&1 || true +sleep 3 +echo "Post-rollback bond0 state:" +grep -E "Bonding Mode|MII Status|Slave Interface|Speed" /proc/net/bonding/bond0 2>/dev/null +echo "Rollback complete." +ROLLBACKEOF +chmod +x "${ROLLBACK}" +echo "Rollback: ${ROLLBACK}" +echo "" + +# ------------------------------------------------------------------------- +# Describe the change +# ------------------------------------------------------------------------- +echo "===================================================================" +echo " PROPOSED CHANGE" +echo "===================================================================" +echo " 1. bond-mode: 802.3ad → active-backup" +echo " 2. Remove bond-xmit-hash-policy (unused in active-backup)" +echo " 3. bond-slaves: keep enx8cae4ccda926 (active NIC only)" +echo "" +echo " Rationale: USB NICs cannot do LACP. ethtool reports Speed: Unknown," +echo " so the bonding driver never sends LACP PDUs (verified via tcpdump:" +echo " 0 LACP PDUs in 65s on both slave and bond master). active-backup" +echo " needs no LACP and works with the single working port." +echo "" +echo " Switch: NO changes needed. g31 is standalone access VLAN 1000." +echo "" + +if [ "${ACTION}" != "apply" ]; then + echo "===================================================================" + echo " DRY RUN — no changes made." + echo " Commit: bash \$0 --apply" + echo "===================================================================" + exit 0 +fi + +# ========================================================================= +# APPLY +# ========================================================================= +echo "===================================================================" +echo " APPLYING — auto-rollback on health check failure" +echo "===================================================================" +echo "" + +# 1. Change bond-mode +echo "Changing bond-mode 802.3ad → active-backup..." +sed -i 's/bond-mode 802\.3ad/bond-mode active-backup/' "${INTERFACES}" + +# 2. Remove bond-xmit-hash-policy (not used by active-backup) +echo "Removing bond-xmit-hash-policy..." +sed -i '/bond-xmit-hash-policy/d' "${INTERFACES}" + +# Show updated stanza +echo "" +echo "--- Updated bond0 stanza ---" +awk '/^auto bond0/,/^$/' "${INTERFACES}" +echo "" + +# 3. Reload networking +echo "--- Reloading networking (ifreload -a) ---" +echo "SSH safe on vmbr0/tailscale0 (not bond0)..." +ifreload -a 2>&1 || echo "WARNING: ifreload returned non-zero — checking state..." + +# Wait for bond to settle +echo "Waiting 5s for bond to settle..." +sleep 5 + +# ========================================================================= +# HEALTH CHECKS +# ========================================================================= +echo "" +echo "===================================================================" +echo " HEALTH CHECKS (5 tests)" +echo "===================================================================" + +HEALTH_OK=true + +# Check 1: bond0 is UP +echo -n " [1/5] bond0 MII up: " +if grep -q "MII Status: up" /proc/net/bonding/bond0 2>/dev/null; then + echo "PASS" +else + echo "FAIL" + HEALTH_OK=false +fi + +# Check 2: Active slave assigned +echo -n " [2/5] Active slave: " +ACTIVE_SLAVE="$(cat /sys/class/net/bond0/bonding/active_slave 2>/dev/null || echo "")" +if [ -n "${ACTIVE_SLAVE}" ]; then + echo "PASS (${ACTIVE_SLAVE})" +else + echo "FAIL (no active slave)" + HEALTH_OK=false +fi + +# Check 3: datanet bridge UP +echo -n " [3/5] datanet bridge up: " +if ip link show datanet 2>/dev/null | grep -q "state UP"; then + echo "PASS" +else + echo "FAIL" + HEALTH_OK=false +fi + +# Check 4: Storage network IP present on datanet +echo -n " [4/5] Storage IP (10.100.100.4): " +if ip addr show datanet 2>/dev/null | grep -q "10.100.100.4"; then + echo "PASS" +else + echo "FAIL" + HEALTH_OK=false +fi + +# Check 5: Storage peer reachable +echo -n " [5/5] Ping ${STORAGE_PEER}: " +if ping -c 3 -W 2 "${STORAGE_PEER}" >/dev/null 2>&1; then + echo "PASS" +else + echo "FAIL" + HEALTH_OK=false +fi + +# ========================================================================= +# COMMIT OR ROLLBACK +# ========================================================================= +echo "" + +if [ "${HEALTH_OK}" = "true" ]; then + echo "===================================================================" + echo " SUCCESS — all 5 health checks passed" + echo "===================================================================" + echo "" + echo "--- Final bond0 state ---" + grep -E "Bonding Mode|MII Status|Slave Interface|Speed|Active" /proc/net/bonding/bond0 + echo "" + echo "Backup: ${BACKUP_DIR}/interfaces" + echo "Rollback: ${ROLLBACK}" + echo "" + echo "Config change is LIVE but not yet reboot-tested." + echo "Verify NFS clients are healthy before next maintenance window." + exit 0 +else + echo "===================================================================" + echo " HEALTH CHECK FAILED — auto-rolling back" + echo "===================================================================" + bash "${ROLLBACK}" + echo "" + echo "Auto-rollback complete." + echo "Original config restored. Manual investigation needed." + exit 1 +fi