diff --git a/WORKING.md b/WORKING.md index f7af686..417d1c0 100644 --- a/WORKING.md +++ b/WORKING.md @@ -5,6 +5,4 @@ A commit is blocked while any task below remains unchecked. ## Current Tasks -- [x] Install iperf3 on all hosts + production VMs (persistent) -- [x] Create reusable any-to-any perf test script -- [x] Enable iperf3 systemd service on hosts for on-demand testing +- [x] Create e1000e offload fix script (repo only, not deployed yet) diff --git a/proxmox/perf/scripts/fix-e1000e-offload.sh b/proxmox/perf/scripts/fix-e1000e-offload.sh new file mode 100755 index 0000000..4e0cd60 --- /dev/null +++ b/proxmox/perf/scripts/fix-e1000e-offload.sh @@ -0,0 +1,79 @@ +#!/bin/bash +############################################################################### +# fix-e1000e-offload.sh — Disable offload on e1000e NICs to prevent hangs +# +# The Intel e1000e driver has a known bug where TSO/GSO/GRO offload causes +# "hardware unit hang" resets on certain Intel NICs (I217-LM, I219-LM, 82571EB). +# This script disables offload features on all e1000e interfaces and installs +# a systemd service to persist across reboots. +# +# References: +# https://forum.proxmox.com/threads/e1000-driver-hang.58284/ +# https://serverfault.com/questions/616485 +# +# Run on each Proxmox host: +# PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file proxmox/perf/scripts/fix-e1000e-offload.sh +############################################################################### +set -euo pipefail + +echo "=== e1000e Offload Fix on $(hostname) ===" + +# Find all e1000e physical NICs (skip bridges, bonds, virtual interfaces) +AFFECTED_NICS=() +for nic_path in /sys/class/net/*; do + nic=$(basename "$nic_path") + [ "$nic" = "lo" ] && continue + # Skip bridges, bonds, virtual interfaces + [ -d "${nic_path}/bridge" ] && continue + [ -d "${nic_path}/bonding" ] && continue + case "$nic" in + tap*|veth*|fwpr*|fwln*|vmbr*|datanet*|storagenet*|tailscale*) continue ;; + esac + + driver=$(ethtool -i "$nic" 2>/dev/null | awk '/^driver:/{print $2}') + if [ "$driver" = "e1000e" ]; then + AFFECTED_NICS+=("$nic") + echo " Found e1000e NIC: $nic" + fi +done + +if [ "${#AFFECTED_NICS[@]}" -eq 0 ]; then + echo " No e1000e NICs found. Nothing to do." + exit 0 +fi + +# Apply fix live +echo "" +echo "--- Disabling offload features ---" +for nic in "${AFFECTED_NICS[@]}"; do + echo " $nic:" + ethtool -K "$nic" tso off gro off gso off tx off rx off 2>&1 | sed 's/^/ /' || true + tso_state=$(ethtool -k "$nic" 2>/dev/null | awk '/tcp-segmentation-offload/{print $2}' | head -1) + echo " tso=$tso_state" +done + +# Install systemd service for persistence +echo "" +echo "--- Installing systemd service ---" +{ + echo "[Unit]" + echo "Description=Disable offload on e1000e NICs (prevent hardware unit hang)" + echo "After=network.target" + echo "Wants=network.target" + echo "" + echo "[Service]" + echo "Type=oneshot" + echo "RemainAfterExit=yes" + for nic in "${AFFECTED_NICS[@]}"; do + echo "ExecStart=/sbin/ethtool -K $nic tso off gro off gso off tx off rx off" + done + echo "" + echo "[Install]" + echo "WantedBy=multi-user.target" +} > /etc/systemd/system/fix-e1000e-offload.service + +systemctl daemon-reload +systemctl enable fix-e1000e-offload.service +echo " Service installed and enabled (fix-e1000e-offload.service)" +echo "" +echo "=== Done. Affected NICs: ${AFFECTED_NICS[*]} ==="