prep for next ai session
This commit is contained in:
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# apply-bond-hash.sh
|
||||
#
|
||||
# Adds bond-xmit-hash-policy layer3+4 to bond0 in /etc/network/interfaces,
|
||||
# then reloads networking with ifreload -a.
|
||||
#
|
||||
# SSH survivability: this is safe IF your SSH session is on vmbr0/nic0
|
||||
# (management network), NOT on bond0/datanet (storage network).
|
||||
# tsys7's topology confirms this: SSH comes in on vmbr0 (nic0).
|
||||
#
|
||||
# Safety:
|
||||
# - Dry-run by default (--apply to commit)
|
||||
# - Full backup of /etc/network/interfaces
|
||||
# - Generates rollback script
|
||||
# - Does NOT reboot — uses ifreload -a which is hot-reload
|
||||
###############################################################################
|
||||
set -euo pipefail
|
||||
|
||||
HOST="$(hostname -s)"
|
||||
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
||||
ROLLBACK="/root/perfopt-bond-rollback-${TS_SHORT}.sh"
|
||||
ACTION="${1:-dryrun}"
|
||||
|
||||
[ "$ACTION" = "--apply" ] && ACTION="apply" || ACTION="dryrun"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " apply-bond-hash — $HOST"
|
||||
echo " mode: $ACTION"
|
||||
echo "==================================================================="
|
||||
|
||||
if [ ! -r /etc/network/interfaces ]; then
|
||||
echo "FATAL: /etc/network/interfaces not readable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if bond0 exists in the config
|
||||
if ! grep -q 'bond0' /etc/network/interfaces; then
|
||||
echo "No bond0 found in /etc/network/interfaces — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Check if the hash policy is already set
|
||||
if grep -q 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces; then
|
||||
echo "bond-xmit-hash-policy already present:"
|
||||
grep 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces
|
||||
echo "Checking value..."
|
||||
if grep -q 'layer3+4' /etc/network/interfaces; then
|
||||
echo "Already set to layer3+4 — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Show current bond state
|
||||
echo ""
|
||||
echo "--- Current bond0 state ---"
|
||||
cat /proc/net/bonding/bond0 2>/dev/null | head -15 || echo "(bond0 not up or not present)"
|
||||
|
||||
# Back up
|
||||
cp -a /etc/network/interfaces "$BACKUP_DIR/interfaces"
|
||||
|
||||
# Generate rollback script
|
||||
cat > "$ROLLBACK" <<EOF
|
||||
#!/bin/bash
|
||||
# Rollback for bond-xmit-hash-policy change
|
||||
# Restores original /etc/network/interfaces and reloads
|
||||
set -euo pipefail
|
||||
cp -a "$BACKUP_DIR/interfaces" /etc/network/interfaces
|
||||
echo "Restored /etc/network/interfaces"
|
||||
echo "Reloading networking..."
|
||||
ifreload -a 2>&1 || systemctl restart networking 2>&1 || true
|
||||
echo "Done. bond0 hash policy reverted to original."
|
||||
EOF
|
||||
chmod +x "$ROLLBACK"
|
||||
|
||||
echo ""
|
||||
echo "--- Proposed change ---"
|
||||
echo "Add line ' bond-xmit-hash-policy layer3+4' to the bond0 stanza."
|
||||
echo ""
|
||||
|
||||
if [ "$ACTION" != "apply" ]; then
|
||||
echo "DRY RUN — no changes made."
|
||||
echo "To commit: bash $0 --apply"
|
||||
echo "Rollback script (pre-generated): $ROLLBACK"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Apply: use sed to insert bond-xmit-hash-policy after bond-mode line
|
||||
# The bond0 stanza looks like:
|
||||
# auto bond0
|
||||
# iface bond0 inet manual
|
||||
# bond-slaves nic1 nic2
|
||||
# bond-miimon 100
|
||||
# bond-mode 802.3ad
|
||||
#
|
||||
# We insert after the bond-mode line.
|
||||
echo "Applying..."
|
||||
|
||||
# Check if bond-mode line exists (various formats)
|
||||
if grep -qE '^\s*bond-mode\s+802.3ad' /etc/network/interfaces; then
|
||||
# Insert after bond-mode 802.3ad line
|
||||
sed -i '/^\s*bond-mode\s+802\.3ad/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode line."
|
||||
elif grep -qE '^\s*bond-mode\s+4' /etc/network/interfaces; then
|
||||
sed -i '/^\s*bond-mode\s+4/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode 4 line."
|
||||
else
|
||||
echo "Could not find bond-mode line — inserting after bond-slaves line instead."
|
||||
sed -i '/^\s*bond-slaves/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
fi
|
||||
|
||||
# Show the result
|
||||
echo ""
|
||||
echo "--- Updated bond0 stanza ---"
|
||||
awk '/^auto bond0/,/^$/' /etc/network/interfaces
|
||||
|
||||
echo ""
|
||||
echo "--- Reloading networking (ifreload -a) ---"
|
||||
echo "SSH should survive (it's on vmbr0/nic0, not bond0)..."
|
||||
ifreload -a 2>&1 || {
|
||||
echo "ifreload failed, trying systemctl restart networking..."
|
||||
systemctl restart networking 2>&1
|
||||
}
|
||||
|
||||
# Wait a moment for bond to renegotiate
|
||||
echo "Waiting 5s for LACP to renegotiate..."
|
||||
sleep 5
|
||||
|
||||
echo ""
|
||||
echo "--- Post-change bond0 state ---"
|
||||
cat /proc/net/bonding/bond0 2>/dev/null | head -20
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " DONE."
|
||||
echo " Backup: $BACKUP_DIR/interfaces"
|
||||
echo " Rollback: bash $ROLLBACK"
|
||||
echo "==================================================================="
|
||||
Executable
+439
@@ -0,0 +1,439 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# apply-tunings.sh
|
||||
#
|
||||
# Applies the Tier 0 host-side tunings identified in FINDINGS.md:
|
||||
# 1. Sets scaling_governor=performance on every CPU
|
||||
# 2. Sets vm.swappiness appropriately (1 on storage hosts, 10 on VM hosts)
|
||||
# 3. Enables tcp_bbr module + sets congestion control
|
||||
# 4. Sets net.core.rmem_max/wmem_max + tcp_rmem/wmem for high-BDP NFS
|
||||
# 5. Sets the recommended tuned-adm profile (network-throughput or virtual-host)
|
||||
# 6. Adds nconnect=4 + noatime to NFS client mounts (edits /etc/pve/storage.cfg)
|
||||
# 7. (does NOT touch bond0 xmit-hash — that requires ifreload/network
|
||||
# restart which drops the host. Provided as a separate --emit-bond-patch
|
||||
# flag that PRINTS the change but does not apply it.)
|
||||
#
|
||||
# Safety features:
|
||||
# - Dry-run mode by default (--apply to commit)
|
||||
# - Full backup of every modified file to /root/perfopt-backup-<timestamp>/
|
||||
# - Per-host behaviour: detects storage hosts by hostname and applies the
|
||||
# right profile (tsys4, tsys5 = storage; others = VM hosts).
|
||||
# - Generates a /root/perfopt-rollback.sh that undoes everything.
|
||||
# - Does NOT reboot, does NOT restart networking, does NOT touch hardware.
|
||||
#
|
||||
# Usage:
|
||||
# bash apply-tunings.sh # dry run, show what would change
|
||||
# bash apply-tunings.sh --apply # commit changes
|
||||
# bash apply-tunings.sh --rollback # restore from latest backup
|
||||
# bash apply-tunings.sh --emit-bond-patch # show bond0 hash change (no apply)
|
||||
###############################################################################
|
||||
set -u
|
||||
umask 022
|
||||
|
||||
HOST="$(hostname -s)"
|
||||
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
||||
ROLLBACK_SCRIPT="/root/perfopt-rollback-${TS_SHORT}.sh"
|
||||
ACTION="dryrun"
|
||||
SKIP_NFS=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--apply) ACTION="apply" ;;
|
||||
--rollback) ACTION="rollback" ;;
|
||||
--emit-bond-patch) ACTION="bond-patch" ;;
|
||||
--no-nfs) SKIP_NFS=1 ;;
|
||||
-h|--help) sed -n '2,35p' "$0"; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Detect host role from hostname
|
||||
case "$HOST" in
|
||||
pfv-tsys4|pfv-tsys5)
|
||||
HOST_ROLE="storage"
|
||||
TUNED_PROFILE="network-throughput"
|
||||
SWAPPINESS="1"
|
||||
;;
|
||||
pfv-tsys1|pfv-tsys3|pfv-tsys6|pfv-tsys7)
|
||||
HOST_ROLE="vmhost"
|
||||
TUNED_PROFILE="virtual-host"
|
||||
SWAPPINESS="10"
|
||||
;;
|
||||
*)
|
||||
HOST_ROLE="unknown"
|
||||
TUNED_PROFILE="virtual-host"
|
||||
SWAPPINESS="10"
|
||||
;;
|
||||
esac
|
||||
|
||||
# --- rollback path ---------------------------------------------------------
|
||||
if [ "$ACTION" = "rollback" ]; then
|
||||
echo "Looking for latest backup under /root/perfopt-backup-* ..."
|
||||
latest=""
|
||||
while IFS= read -r d; do
|
||||
latest="$d"
|
||||
done < <(find /root -maxdepth 1 -type d -name 'perfopt-backup-*' 2>/dev/null | sort | tail -n1)
|
||||
if [ -z "$latest" ]; then
|
||||
echo "No backup found under /root/perfopt-backup-*" >&2
|
||||
exit 1
|
||||
fi
|
||||
rb="$latest/perfopt-rollback.sh"
|
||||
if [ ! -x "$rb" ] && [ ! -r "$rb" ]; then
|
||||
echo "Rollback script missing in $latest" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Rolling back using: $rb"
|
||||
bash "$rb"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# --- bond patch print-only path -------------------------------------------
|
||||
if [ "$ACTION" = "bond-patch" ]; then
|
||||
echo "==================================================================="
|
||||
echo " Proposed bond0 xmit_hash_policy change"
|
||||
echo "==================================================================="
|
||||
echo
|
||||
if [ ! -r /etc/network/interfaces ]; then
|
||||
echo "/etc/network/interfaces not readable"
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q 'bond-mode 802.3ad\|bond-mode 4\|bond-slaves' /etc/network/interfaces; then
|
||||
echo "No bond0 detected on this host — nothing to patch."
|
||||
exit 0
|
||||
fi
|
||||
cat <<EOF
|
||||
A manual edit to /etc/network/interfaces is required. The bond0 stanza
|
||||
needs this line added (it defaults to layer2, which is wrong for storage):
|
||||
|
||||
bond-xmit-hash-policy layer3+4
|
||||
|
||||
After editing, you MUST reload networking for it to take effect:
|
||||
ifreload -a # safe, brings interfaces down/up
|
||||
# OR
|
||||
systemctl restart networking # heavier, briefly drops connections
|
||||
|
||||
WARNING: applying this on a remote host over bond0 will briefly drop your
|
||||
SSH session. Run from console/IPMI, or schedule a maintenance window.
|
||||
|
||||
The change is reversible by removing the line and reloading again.
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- main path (dryrun or apply) ------------------------------------------
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Emit the rollback script header
|
||||
cat > "$ROLLBACK_SCRIPT" <<EOF
|
||||
#!/bin/bash
|
||||
# Auto-generated rollback for perfopt apply-tunings.sh
|
||||
# Backup timestamp: $TS
|
||||
# Backup dir: $BACKUP_DIR
|
||||
# Generated on: $HOST
|
||||
set -u
|
||||
EOF
|
||||
chmod +x "$ROLLBACK_SCRIPT"
|
||||
|
||||
backup_file() {
|
||||
local f="$1"
|
||||
if [ -e "$f" ]; then
|
||||
cp -a "$f" "$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
||||
fi
|
||||
}
|
||||
|
||||
# Helper: append a restore command to the rollback script
|
||||
rb_restore() {
|
||||
local f="$1"
|
||||
local bk
|
||||
bk="$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
if [ -r "$bk" ]; then
|
||||
cp -a "$bk" "$f" && echo "restored $f"
|
||||
else
|
||||
echo "WARN: backup $bk missing — $f left as-is"
|
||||
fi
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: append a sysctl restore command
|
||||
rb_sysctl_restore() {
|
||||
local key="$1" orig_val="$2"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
sysctl -w "$key=$orig_val" >/dev/null && echo "restored $key=$orig_val"
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: append a tuned-adm restore
|
||||
rb_tuned_restore() {
|
||||
local profile="$1"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
tuned-adm profile "$profile" 2>/dev/null && echo "restored tuned profile: $profile"
|
||||
EOF
|
||||
}
|
||||
|
||||
echo "==================================================================="
|
||||
echo " perfopt apply-tunings — $HOST"
|
||||
echo " mode: $ACTION"
|
||||
echo " role: $HOST_ROLE (tuned=$TUNED_PROFILE, swappiness=$SWAPPINESS)"
|
||||
echo " backup: $BACKUP_DIR"
|
||||
echo " rollback: $ROLLBACK_SCRIPT"
|
||||
echo "==================================================================="
|
||||
echo
|
||||
|
||||
# ===========================================================================
|
||||
# 1. scaling_governor → performance
|
||||
# ===========================================================================
|
||||
echo "--- 1. CPU scaling_governor → performance"
|
||||
if [ -r /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
|
||||
cur=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
|
||||
echo " current: $cur"
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
if [ "$cur" != "performance" ]; then
|
||||
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||
echo performance > "$c" 2>/dev/null || true
|
||||
done
|
||||
# Persist via systemd tmpfiles / sysctl fallback
|
||||
cat > /etc/systemd/system/perfopt-cpu-performance.service <<EOF
|
||||
[Unit]
|
||||
Description=Set CPU scaling_governor=performance (perfopt)
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/sh -c 'for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "\$c" 2>/dev/null || true; done'
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
systemctl enable perfopt-cpu-performance.service 2>/dev/null
|
||||
systemctl start perfopt-cpu-performance.service 2>/dev/null
|
||||
echo " applied + persisted via systemd unit"
|
||||
# Rollback
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "$cur" > "\$c" 2>/dev/null || true; done
|
||||
systemctl disable --now perfopt-cpu-performance.service 2>/dev/null || true
|
||||
rm -f /etc/systemd/system/perfopt-cpu-performance.service
|
||||
systemctl daemon-reload
|
||||
echo "restored scaling_governor=$cur (best-effort; original may have been dynamic)"
|
||||
EOF
|
||||
else
|
||||
echo " already performance — no change"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " cpufreq driver not loaded — nothing to do (typical on BIOS-locked servers)"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 2. vm.swappiness
|
||||
# ===========================================================================
|
||||
echo "--- 2. vm.swappiness → $SWAPPINESS (role=$HOST_ROLE)"
|
||||
cur_swappiness=$(sysctl -n vm.swappiness 2>/dev/null || echo "?")
|
||||
echo " current: $cur_swappiness"
|
||||
if [ "$ACTION" = "apply" ] && [ "$cur_swappiness" != "$SWAPPINESS" ]; then
|
||||
# Persist via sysctl.d
|
||||
backup_file /etc/sysctl.d/99-perfopt.conf
|
||||
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
||||
vm.swappiness = $SWAPPINESS
|
||||
EOF
|
||||
sysctl -w "vm.swappiness=$SWAPPINESS" >/dev/null
|
||||
echo " applied (persisted to /etc/sysctl.d/99-perfopt.conf)"
|
||||
rb_restore /etc/sysctl.d/99-perfopt.conf
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
rm -f /etc/sysctl.d/99-perfopt.conf
|
||||
sysctl -w "vm.swappiness=$cur_swappiness" >/dev/null
|
||||
echo "restored vm.swappiness=$cur_swappiness"
|
||||
EOF
|
||||
else
|
||||
echo " no change"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 3. TCP BBR + buffers (single sysctl.d file)
|
||||
# ===========================================================================
|
||||
echo "--- 3. TCP BBR + socket buffers"
|
||||
cur_cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo "?")
|
||||
cur_avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null || echo "?")
|
||||
echo " current: $cur_cc (available: $cur_avail)"
|
||||
|
||||
# Decide sysctl values (these are the consensus values for a 1-10 GbE NFS host)
|
||||
TARGET_RMEM_MAX=134217728 # 128 MB
|
||||
TARGET_WMEM_MAX=134217728
|
||||
TARGET_RMEM_DEFAULT=26214400 # 25 MB
|
||||
TARGET_WMEM_DEFAULT=26214400
|
||||
TARGET_TCP_RMEM="4096 87380 $TARGET_RMEM_MAX"
|
||||
TARGET_TCP_WMEM="4096 65536 $TARGET_RMEM_MAX"
|
||||
TARGET_NETDEV_MAX_BACKLOG=250000
|
||||
TARGET_SOMAXCONN=65535
|
||||
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
# Ensure tcp_bbr module loads at boot
|
||||
if ! grep -q '^tcp_bbr' /etc/modules-load.d/modules.conf 2>/dev/null; then
|
||||
backup_file /etc/modules-load.d/modules.conf
|
||||
mkdir -p /etc/modules-load.d
|
||||
cat >> /etc/modules-load.d/modules.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS
|
||||
tcp_bbr
|
||||
EOF
|
||||
modprobe tcp_bbr 2>/dev/null || true
|
||||
rb_restore /etc/modules-load.d/modules.conf
|
||||
fi
|
||||
|
||||
backup_file /etc/sysctl.d/99-perfopt.conf
|
||||
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
||||
vm.swappiness = $SWAPPINESS
|
||||
|
||||
# TCP BBR + high-BDP buffers (good for NFS over 1-10 GbE)
|
||||
net.ipv4.tcp_congestion_control = bbr
|
||||
net.core.default_qdisc = fq
|
||||
net.core.rmem_max = $TARGET_RMEM_MAX
|
||||
net.core.wmem_max = $TARGET_WMEM_MAX
|
||||
net.core.rmem_default = $TARGET_RMEM_DEFAULT
|
||||
net.core.wmem_default = $TARGET_WMEM_DEFAULT
|
||||
net.core.netdev_max_backlog = $TARGET_NETDEV_MAX_BACKLOG
|
||||
net.core.somaxconn = $TARGET_SOMAXCONN
|
||||
net.ipv4.tcp_rmem = $TARGET_TCP_RMEM
|
||||
net.ipv4.tcp_wmem = $TARGET_TCP_WMEM
|
||||
EOF
|
||||
sysctl --system >/dev/null 2>&1
|
||||
echo " applied (written to /etc/sysctl.d/99-perfopt.conf + sysctl --system)"
|
||||
# Rebuild rollback for sysctl
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
# Restore original sysctl state
|
||||
rm -f /etc/sysctl.d/99-perfopt.conf
|
||||
EOF
|
||||
rb_sysctl_restore "net.ipv4.tcp_congestion_control" "$cur_cc"
|
||||
rb_sysctl_restore "vm.swappiness" "$cur_swappiness"
|
||||
# Note: original buffer sizes not all captured; restoration is best-effort.
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
echo "Note: net buffers restored to kernel defaults (original values not captured)."
|
||||
echo "Run 'sysctl --system' to apply."
|
||||
sysctl --system >/dev/null 2>&1 || true
|
||||
EOF
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 4. tuned-adm profile
|
||||
# ===========================================================================
|
||||
echo "--- 4. tuned-adm profile → $TUNED_PROFILE"
|
||||
if command -v tuned-adm >/dev/null 2>&1; then
|
||||
cur_profile=$(tuned-adm active 2>/dev/null | awk -F: '/Current active profile/{gsub(/^[ \t]+/,"",$2); print $2}')
|
||||
echo " current: $cur_profile"
|
||||
if [ "$ACTION" = "apply" ] && [ "$cur_profile" != "$TUNED_PROFILE" ]; then
|
||||
tuned-adm profile "$TUNED_PROFILE" 2>&1 | sed 's/^/ /'
|
||||
rb_tuned_restore "$cur_profile"
|
||||
else
|
||||
echo " no change"
|
||||
fi
|
||||
else
|
||||
echo " tuned-adm not installed — skipping"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 5. NFS mount option hardening
|
||||
# Edit /etc/pve/storage.cfg to add nconnect=4 + noatime to NFS plugins.
|
||||
# Proxmox applies these on next mount/remount.
|
||||
# Skipped with --no-nfs (for NFS servers where we don't want to remount).
|
||||
# ===========================================================================
|
||||
echo "--- 5. NFS mount options: add nconnect=4, noatime"
|
||||
if [ "$SKIP_NFS" -eq 1 ]; then
|
||||
echo " SKIPPED (--no-nfs specified)"
|
||||
echo " Note: storage.cfg is cluster-wide; if edited on another host,"
|
||||
echo " the options are already staged here and activate on next reboot."
|
||||
else
|
||||
if [ -r /etc/pve/storage.cfg ]; then
|
||||
backup_file /etc/pve/storage.cfg
|
||||
if grep -q '^nfs:' /etc/pve/storage.cfg; then
|
||||
# Count how many nfs: stanzas lack the options
|
||||
while IFS= read -r line; do
|
||||
if echo "$line" | grep -q '^nfs:'; then
|
||||
cur_stanza=$(echo "$line" | awk '{print $2}')
|
||||
# Look ahead for the next few lines to see if options already set
|
||||
echo " nfs stanza: $cur_stanza"
|
||||
fi
|
||||
done < /etc/pve/storage.cfg
|
||||
|
||||
# Check whether any nfs stanza already has options
|
||||
if grep -A1 '^nfs:' /etc/pve/storage.cfg | grep -q 'options.*nconnect=4'; then
|
||||
echo " some stanzas already have nconnect=4 — check manually"
|
||||
else
|
||||
echo " proposed change: add 'options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2' to each nfs stanza"
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
# Apply via perl for safety (in-place edit with backup)
|
||||
if cp -a /etc/pve/storage.cfg "$BACKUP_DIR/etc__pve__storage.cfg"; then
|
||||
# Use a python helper for robust PVE storage.cfg editing
|
||||
python3 - <<PYEOF
|
||||
import re, pathlib
|
||||
p = pathlib.Path("/etc/pve/storage.cfg")
|
||||
text = p.read_text()
|
||||
pattern = re.compile(r'(^nfs:\s*\S+\n(?:[ \t]+[^\n]+\n)+)', re.MULTILINE)
|
||||
modified = 0
|
||||
def fix(m):
|
||||
global modified
|
||||
block = m.group(1)
|
||||
if 'options' in block:
|
||||
return block
|
||||
lines = block.rstrip('\n').split('\n')
|
||||
lines.insert(1, '\toptions nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2')
|
||||
modified += 1
|
||||
return '\n'.join(lines) + '\n'
|
||||
new_text = pattern.sub(fix, text)
|
||||
if modified:
|
||||
p.write_text(new_text)
|
||||
print(f" patched {modified} NFS stanzas with options line")
|
||||
else:
|
||||
print(" no NFS stanza needed patching (all already had options or no nfs: found)")
|
||||
PYEOF
|
||||
# Remount existing NFS mounts to pick up new options
|
||||
echo " remounting NFS mounts to apply new options..."
|
||||
while IFS= read -r mp; do
|
||||
mount -o remount "$mp" 2>/dev/null && echo " remounted $mp" || echo " FAILED to remount $mp (will pick up on next mount)"
|
||||
done < <(awk '$3 ~ /^nfs/{print $2}' /proc/mounts 2>/dev/null | sort -u)
|
||||
rb_restore /etc/pve/storage.cfg
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
# To fully undo NFS options, also remount:
|
||||
while IFS= read -r mp; do
|
||||
mount -o remount "\$mp" 2>/dev/null
|
||||
done < <(awk '\$3 ~ /^nfs/{print \$2}' /proc/mounts 2>/dev/null | sort -u)
|
||||
echo "restored NFS mount options"
|
||||
EOF
|
||||
else
|
||||
echo " ERROR: could not back up storage.cfg — aborting NFS patch"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " no NFS stanzas in storage.cfg — skipping"
|
||||
fi
|
||||
else
|
||||
echo " /etc/pve/storage.cfg not readable — skipping"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 6. Summary
|
||||
# ===========================================================================
|
||||
echo
|
||||
echo "==================================================================="
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
echo " DONE. Backups in: $BACKUP_DIR"
|
||||
echo " Rollback: bash $ROLLBACK_SCRIPT"
|
||||
echo
|
||||
echo " Next steps:"
|
||||
echo " - Verify with 'sysctl net.ipv4.tcp_congestion_control vm.swappiness'"
|
||||
echo " - Verify NFS mounts with 'nfsstat -m' (look for nconnect=4)"
|
||||
echo " - Run 'bash apply-tunings.sh --emit-bond-patch' for the bond0"
|
||||
echo " xmit_hash_policy change (requires network restart, do in"
|
||||
echo " maintenance window)"
|
||||
else
|
||||
echo " DRY RUN — no changes made."
|
||||
echo " To commit: bash apply-tunings.sh --apply"
|
||||
fi
|
||||
echo "==================================================================="
|
||||
Executable
+1138
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
conman-console.py — Drive a serial console via a conman server, read-only.
|
||||
|
||||
Connects to a conmand server (RFC 127-style multiplexer) over the network,
|
||||
opens a named console, sends the commands from a .cmds file, captures all
|
||||
output, and disconnects. Requires no expect/tcl — pure stdlib PTY.
|
||||
|
||||
This replaces the old sw-capture-remote.sh workflow that killed the serial
|
||||
device holder (conflicting with conman/ser2net). Instead, it talks to conman
|
||||
over TCP, which multiplexes safely with other sessions.
|
||||
|
||||
All endpoints are configurable via environment variables so this works on
|
||||
any network with a conman server:
|
||||
|
||||
CONMAN_SERVER conman server host:port (default: via CONSOLE_HOST)
|
||||
CONSOLE console name to open (required)
|
||||
CMDS_FILE file of commands to send (required)
|
||||
TIMEOUT overall timeout in seconds (default: 45)
|
||||
CMD_DELAY seconds between commands (default: 3)
|
||||
WAKE_DELAY seconds after connect (default: 2)
|
||||
|
||||
Usage:
|
||||
CONMAN_SERVER=console-host:7890 \\
|
||||
python3 conman-console.py --console pfv-core-sw01 --cmds switches/pfv-core-sw01.cmds
|
||||
|
||||
Lines starting with '!' or '#' in the cmds file are comments (skipped).
|
||||
Blank lines are skipped. The conman escape sequence (&.) is sent automatically
|
||||
to disconnect. A password prompt aborts immediately (we never send creds).
|
||||
|
||||
Exit codes:
|
||||
0 clean run
|
||||
1 usage / setup error
|
||||
2 could not connect to conman server
|
||||
3 timeout (partial output still printed)
|
||||
4 password prompt encountered (aborted)
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import pty
|
||||
import re
|
||||
import select
|
||||
import sys
|
||||
import time
|
||||
|
||||
PWD_RE = re.compile(rb"[Pp]assword:\s*$")
|
||||
MORE_RE = re.compile(rb"--\s*[Mm]ore\s*--|[Mm]ore:\s*<space>")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Drive a conman console session read-only via PTY")
|
||||
ap.add_argument("--console", required=True,
|
||||
help="console name (e.g. pfv-core-sw01)")
|
||||
ap.add_argument("--cmds", required=True,
|
||||
help="command file (one command per line; !/# = comment)")
|
||||
ap.add_argument("--server",
|
||||
default=os.environ.get("CONMAN_SERVER", ""),
|
||||
help="conman server host:port (env: CONMAN_SERVER)")
|
||||
ap.add_argument("--timeout", type=int,
|
||||
default=int(os.environ.get("TIMEOUT", "45")),
|
||||
help="overall timeout seconds (env: TIMEOUT)")
|
||||
ap.add_argument("--cmd-delay", type=float,
|
||||
default=float(os.environ.get("CMD_DELAY", "3")),
|
||||
help="seconds between commands (env: CMD_DELAY)")
|
||||
ap.add_argument("--wake-delay", type=float,
|
||||
default=float(os.environ.get("WAKE_DELAY", "2")),
|
||||
help="seconds after connect before first command (env: WAKE_DELAY)")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.server:
|
||||
sys.stderr.write("ERROR: --server or CONMAN_SERVER env required\n")
|
||||
return 1
|
||||
|
||||
with open(args.cmds) as f:
|
||||
cmds = [l.strip() for l in f
|
||||
if l.strip() and not l.strip().startswith(("!", "#"))]
|
||||
|
||||
pid, fd = pty.fork()
|
||||
if pid == 0:
|
||||
try:
|
||||
os.execvp("conman",
|
||||
["conman", "-d", args.server, "-f", args.console])
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"ERROR: cannot exec conman: {e}\n")
|
||||
os._exit(2)
|
||||
os._exit(2)
|
||||
|
||||
output = b""
|
||||
cmd_queue = list(cmds)
|
||||
sent_disconnect = False
|
||||
start = time.time()
|
||||
last_action = 0.0
|
||||
phase = "connect"
|
||||
|
||||
while time.time() - start < args.timeout:
|
||||
ready, _, _ = select.select([fd], [], [], 0.5)
|
||||
if ready:
|
||||
try:
|
||||
data = os.read(fd, 8192)
|
||||
except OSError:
|
||||
break
|
||||
if not data:
|
||||
break
|
||||
output += data
|
||||
|
||||
if PWD_RE.search(output.split(b"\n")[-1] if output else b""):
|
||||
sys.stderr.write("[ABORT] password prompt detected — "
|
||||
"never sending credentials\n")
|
||||
os.write(fd, b"&.\n")
|
||||
break
|
||||
|
||||
# Handle pagination: send space to continue
|
||||
if MORE_RE.search(output[-200:] if output else b""):
|
||||
os.write(fd, b" ")
|
||||
time.sleep(0.5)
|
||||
|
||||
elapsed = time.time() - start
|
||||
gap = elapsed - last_action
|
||||
|
||||
if phase == "connect" and gap >= args.wake_delay:
|
||||
os.write(fd, b"\n")
|
||||
phase = "send"
|
||||
last_action = elapsed
|
||||
elif phase == "send" and gap >= args.cmd_delay:
|
||||
if cmd_queue:
|
||||
cmd = cmd_queue.pop(0)
|
||||
os.write(fd, (cmd + "\n").encode())
|
||||
last_action = elapsed
|
||||
else:
|
||||
phase = "drain"
|
||||
last_action = elapsed
|
||||
elif phase == "drain" and gap >= args.cmd_delay:
|
||||
os.write(fd, b"&.\n")
|
||||
sent_disconnect = True
|
||||
phase = "done"
|
||||
last_action = elapsed
|
||||
elif phase == "done" and gap >= 2:
|
||||
break
|
||||
|
||||
if not sent_disconnect:
|
||||
try:
|
||||
os.write(fd, b"&.\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.waitpid(pid, 0)
|
||||
except ChildProcessError:
|
||||
pass
|
||||
|
||||
sys.stdout.buffer.write(output)
|
||||
sys.stdout.flush()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
# fix-tsys7.sh - fixes the two issues found: NFS options string + bond hash
|
||||
# Runs on the target host directly.
|
||||
set -euo pipefail
|
||||
|
||||
echo "==================================================================="
|
||||
echo " FIX: NFS options + bond hash policy"
|
||||
echo "==================================================================="
|
||||
|
||||
# --- FIX 1: Simplify NFS options in storage.cfg ---------------------------
|
||||
echo ""
|
||||
echo "--- FIX 1: Simplify NFS options (remove version=4.2 conflict) ---"
|
||||
# Replace the overly-complex options line with a minimal one
|
||||
# PVE handles vers/rsize/wsize/hard/etc internally; we only need nconnect + noatime
|
||||
if grep -q 'options.*nconnect=4.*version=4.2' /etc/pve/storage.cfg; then
|
||||
# Use sed to replace each options line
|
||||
sed -i 's/options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2/options nconnect=4,noatime/' /etc/pve/storage.cfg
|
||||
echo "Fixed: simplified NFS options to 'nconnect=4,noatime'"
|
||||
else
|
||||
echo "Options line already simplified or not present"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Verify storage.cfg NFS stanzas ---"
|
||||
grep -A 2 "^nfs: D2" /etc/pve/storage.cfg | head -3
|
||||
echo "..."
|
||||
grep -c "options nconnect" /etc/pve/storage.cfg
|
||||
echo " NFS stanzas with options"
|
||||
|
||||
# --- FIX 2: Bond hash policy (force via sysfs + persist in interfaces) ----
|
||||
echo ""
|
||||
echo "--- FIX 2: Apply bond xmit_hash_policy=layer3+4 ---"
|
||||
|
||||
# 2a: Apply LIVE via sysfs (takes effect immediately, no network reload)
|
||||
echo "Applying live via sysfs..."
|
||||
if echo "layer3+4" > /sys/class/net/bond0/bonding/xmit_hash_policy 2>/dev/null; then
|
||||
echo "Live sysfs apply: SUCCESS"
|
||||
else
|
||||
echo "Live sysfs apply: FAILED (will persist in config and apply on ifreload)"
|
||||
fi
|
||||
|
||||
# Verify live state
|
||||
echo ""
|
||||
echo "Live bond0 hash policy:"
|
||||
cat /proc/net/bonding/bond0 | grep "Transmit Hash"
|
||||
|
||||
# 2b: Persist in /etc/network/interfaces (fix the sed that failed before)
|
||||
echo ""
|
||||
echo "Persisting in /etc/network/interfaces..."
|
||||
|
||||
# Backup
|
||||
cp -a /etc/network/interfaces "/root/interfaces.bondfix.$(date +%Y%m%d%H%M%S)"
|
||||
|
||||
# Check if already present
|
||||
if grep -q 'bond-xmit-hash-policy' /etc/network/interfaces; then
|
||||
echo "bond-xmit-hash-policy already in interfaces file"
|
||||
else
|
||||
# Use awk to insert after the bond-mode line (more reliable than sed)
|
||||
# Match any line containing 'bond-mode' (regardless of indentation)
|
||||
awk '
|
||||
/bond-mode/ && !done {
|
||||
print
|
||||
print "\tbond-xmit-hash-policy layer3+4"
|
||||
done=1
|
||||
next
|
||||
}
|
||||
{ print }
|
||||
' /etc/network/interfaces > /etc/network/interfaces.new
|
||||
|
||||
mv /etc/network/interfaces.new /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Updated bond0 stanza ---"
|
||||
awk '/^auto bond0/,/^$/' /etc/network/interfaces
|
||||
|
||||
# Final verify
|
||||
echo ""
|
||||
echo "--- Final bond0 running state ---"
|
||||
cat /proc/net/bonding/bond0 | head -20
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " FIX COMPLETE"
|
||||
echo "==================================================================="
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
# identify-hosts.sh — identify all live hosts on a network via DNS/SNMP/SSH
|
||||
# Usage: bash identify-hosts.sh <hosts-file-or-subnet>
|
||||
# bash identify-hosts.sh 192.168.0.0/22
|
||||
# bash identify-hosts.sh /tmp/live-hosts.txt
|
||||
set -u
|
||||
COMMUNITY="${SNMP_COMMUNITY:-kn3lmgmt}"
|
||||
|
||||
if [ -f "${1:-}" ]; then
|
||||
HOSTS=$(cat "$1")
|
||||
elif [ -n "${1:-}" ]; then
|
||||
HOSTS=$(nmap -sn -n "$1" 2>/dev/null | grep 'Nmap scan report' | awk '{print $5}')
|
||||
else
|
||||
echo "Usage: $0 <hosts-file-or-subnet>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "%-16s %-30s %-25s %-20s\n" "IP" "DNS_NAME" "SNMP_SYSDESCR" "SSH_BANNER"
|
||||
printf "%-16s %-30s %-25s %-20s\n" "--" "-------" "-------------" "----------"
|
||||
|
||||
for ip in $HOSTS; do
|
||||
# DNS reverse
|
||||
rdns=$(dig +short -x "$ip" 2>/dev/null | head -1 | sed 's/\.$//')
|
||||
[ -z "$rdns" ] && rdns="(no-ptr)"
|
||||
|
||||
# SNMP sysDescr (first 25 chars)
|
||||
snmp=$(timeout 3 snmpget -Oqv -v2c -c "$COMMUNITY" "$ip" 1.3.6.1.2.1.1.1.0 2>/dev/null | head -1 | tr -d '"' | cut -c1-25)
|
||||
[ -z "$snmp" ] && snmp="-"
|
||||
|
||||
# SSH banner (quick)
|
||||
ssh_banner=$(timeout 3 bash -c "echo > /dev/tcp/$ip/22" 2>/dev/null && echo "ssh:open" || echo "-")
|
||||
|
||||
printf "%-16s %-30s %-25s %-20s\n" "$ip" "$rdns" "$snmp" "$ssh_banner"
|
||||
done
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# lacp-retrans-cause.sh (HOST-NATIVE)
|
||||
#
|
||||
# Runs on tsys6 (receiver) or tsys7 (sender). Auto-detects role.
|
||||
#
|
||||
# Question this answers: are the ~56K retransmits we see on 8-stream iperf
|
||||
# between tsys6 and tsys7 fixable (NIC ring drops, softnet drops, CPU
|
||||
# saturation) or just unavoidable LACP reordering overhead?
|
||||
#
|
||||
# Method: snapshot drop/error counters + softnet_stat + TCP SNMP before and
|
||||
# after a 12s iperf3 run, then print the deltas. The decisive columns are:
|
||||
# - NIC rx_dropped / rx_missed_errors / rx_no_dma_resources -> ring too small
|
||||
# - /proc/net/softnet_stat drops -> ksoftirq starved
|
||||
# - /proc/net/snmp TCP retranst -> TCP-level retrans
|
||||
# If NIC drop counters do NOT climb but TCP retrans does, the retrans are
|
||||
# coming from LACP reordering (out-of-order segments triggering fast
|
||||
# retransmit), not packet loss — and are NOT fixable by tuning.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
DURATION="${LACP_TEST_SECS:-12}"
|
||||
STREAMS="${LACP_STREAMS:-8}"
|
||||
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
||||
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
||||
LOG_DIR="/root"
|
||||
BOND="bond0"
|
||||
|
||||
HOST="$(uname -n)"; HOST="${HOST%%.*}"
|
||||
case "$HOST" in
|
||||
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-retrans-receiver.log" ;;
|
||||
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-retrans-sender.log" ;;
|
||||
*) echo "ERROR: not on tsys6 or tsys7 (host=$HOST)"; exit 2 ;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# ---- per-slave NIC counter snapshot -----------------------------------------
|
||||
# Captures rx_dropped, rx_missed_errors, rx_no_dma_resources, tx_retrans,
|
||||
# plus all error-like counters. Falls back gracefully if a counter doesn't
|
||||
# exist (different NIC drivers expose different names).
|
||||
nic_snapshot() {
|
||||
# shellcheck disable=SC2013 # interface names contain no spaces; word-splitting is safe here
|
||||
for s in $(awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND"); do
|
||||
[ -n "$s" ] || continue
|
||||
echo "[$s]"
|
||||
# ethtool -S may exist; show only drop/error/retrans lines
|
||||
if command -v ethtool >/dev/null 2>&1; then
|
||||
ethtool -S "$s" 2>/dev/null \
|
||||
| grep -iE 'drop|miss|error|no_dma|fifo|retrans|overflow' \
|
||||
|| echo "(ethtool -S: no matching counters or unsupported)"
|
||||
else
|
||||
echo "(ethtool not installed)"
|
||||
fi
|
||||
# /sys counters (always available, driver-agnostic)
|
||||
for c in rx_dropped tx_dropped rx_errors tx_errors rx_missed_errors \
|
||||
rx_length_errors rx_crc_errors rx_fifo_errors tx_fifo_errors \
|
||||
multicast collisions; do
|
||||
v=$(cat "/sys/class/net/$s/statistics/$c" 2>/dev/null)
|
||||
[ -n "$v" ] && printf ' sysfs %-22s = %s\n' "$c" "$v"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# ---- softnet_stat snapshot (per-CPU RX softirq drops) -----------------------
|
||||
# /proc/net/softnet_stat columns are HEX. Col1=processed, col2=dropped,
|
||||
# col3=time_squeeze. We only care about dropped + squeeze (small numbers that
|
||||
# look identical in hex and decimal). Processed is informational only and we
|
||||
# don't try to delta it (hex arithmetic is awk-version-dependent).
|
||||
softnet_snapshot() {
|
||||
awk '{ printf "cpu%s processed=%s dropped=%s squeezed=%s\n", \
|
||||
NR-1, $1, $2, $3 }' /proc/net/softnet_stat
|
||||
}
|
||||
|
||||
# ---- TCP SNMP snapshot (the source of iperf's retransmit number) -----------
|
||||
tcp_snapshot() {
|
||||
awk '/^Tcp:/{
|
||||
if (!seen) {
|
||||
seen=1
|
||||
# /proc/net/snmp Tcp line 1 = names, line 2 = values
|
||||
n=split($0, names, " ")
|
||||
# re-find the values line
|
||||
getline
|
||||
vals=$0
|
||||
split(vals, v, " ")
|
||||
for (i=1;i<=n;i++) printf " %-22s = %s\n", names[i], v[i]
|
||||
}
|
||||
}' /proc/net/snmp
|
||||
}
|
||||
|
||||
# ---- print deltas for selected counters ------------------------------------
|
||||
# $1 = before file, $2 = after file
|
||||
# softnet_snapshot output: "cpuN processed=HEX dropped=HEX squeezed=HEX"
|
||||
# /proc/net/softnet_stat is hex, but dropped/squeezed are always small
|
||||
# integers (typically 0 on a healthy host), so +0 coercion is correct for
|
||||
# the values we care about. We deliberately DON'T report `processed` deltas
|
||||
# because hex arithmetic on large numbers is awk-version-dependent.
|
||||
softnet_delta() {
|
||||
awk '
|
||||
FNR==NR { if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
||||
split($0, p, /[= ]+/); drop[id]=p[5]+0; sqz[id]=p[7]+0 }
|
||||
next }
|
||||
{ if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
||||
split($0, q, /[= ]+/);
|
||||
dd = (q[5]+0) - drop[id]
|
||||
sd = (q[7]+0) - sqz[id]
|
||||
if (dd != 0 || sd != 0)
|
||||
printf " %-8s dropped_delta=%-6s squeezed_delta=%-6s\n", id, dd, sd } }
|
||||
' "$1" "$2" | sort
|
||||
}
|
||||
|
||||
# $1 = before file, $2 = after file, $3 = section label prefix
|
||||
# File format: slave header line "[nic1]", then " counter_name = value" lines.
|
||||
# Output only counters whose value changed.
|
||||
nic_delta() {
|
||||
awk '
|
||||
FNR==NR { if ($0 ~ /^\[/) slave=$0
|
||||
else if ($0 ~ /=/) {
|
||||
# Everything before " = " is the counter name (trim spaces)
|
||||
pos = index($0, "=")
|
||||
name = substr($0, 1, pos-1)
|
||||
gsub(/^ +| +$/, "", name)
|
||||
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
||||
before[slave SUBSEP name] = val
|
||||
}
|
||||
next }
|
||||
{ if ($0 ~ /^\[/) slave=$0
|
||||
else if ($0 ~ /=/) {
|
||||
pos = index($0, "=")
|
||||
name = substr($0, 1, pos-1); gsub(/^ +| +$/, "", name)
|
||||
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
||||
b = before[slave SUBSEP name]+0
|
||||
d = val+0 - b
|
||||
if (d != 0) printf " %-8s %-30s %12s -> %12s delta=%d\n", slave, name, b, val, d
|
||||
} }
|
||||
' "$1" "$2"
|
||||
}
|
||||
|
||||
{
|
||||
echo "=== LACP retransmit cause investigation ($ROLE) ==="
|
||||
echo "Host: $HOST Role: $ROLE"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Bond: $BOND"
|
||||
echo ""
|
||||
echo "--- bond0 hash + driver ---"
|
||||
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac" /proc/net/bonding/$BOND
|
||||
# shellcheck disable=SC2013 # interface names contain no spaces; word-splitting is safe here
|
||||
for s in $(awk '/^Slave Interface:/{print $3}' /proc/net/bonding/$BOND); do
|
||||
drv=$(ethtool -i "$s" 2>/dev/null | awk -F: '/^driver:/{print $2}' | sed 's/^ *//')
|
||||
speed=$(cat "/sys/class/net/$s/speed" 2>/dev/null)
|
||||
ring=$(ethtool -g "$s" 2>/dev/null | awk '/RX:/{print $2; exit}')
|
||||
printf " %-8s driver=%-20s speed=%-6s current RX ring=%s\n" "$s" "$drv" "$speed" "$ring"
|
||||
done
|
||||
echo ""
|
||||
} | tee "$LOG"
|
||||
|
||||
echo "--- BEFORE snapshots ---"
|
||||
nic_snapshot > "$TMP/nic_before"
|
||||
softnet_snapshot > "$TMP/softnet_before"
|
||||
tcp_snapshot > "$TMP/tcp_before"
|
||||
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
echo "[receiver] starting iperf3 -s -1 on $PEER_RX_IP ..."
|
||||
pkill -x iperf3 2>/dev/null; sleep 0.5
|
||||
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
||||
SRV_PID=$!
|
||||
for _ in $(seq 1 60); do kill -0 "$SRV_PID" 2>/dev/null || break; sleep 1; done
|
||||
wait "$SRV_PID" 2>/dev/null
|
||||
else
|
||||
echo "[sender] waiting 3s for receiver to listen, then running iperf3 -P $STREAMS -t $DURATION ..."
|
||||
sleep 3
|
||||
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
||||
> "$TMP/iperf.out" 2>&1
|
||||
fi
|
||||
|
||||
echo "--- AFTER snapshots ---"
|
||||
nic_snapshot > "$TMP/nic_after"
|
||||
softnet_snapshot > "$TMP/softnet_after"
|
||||
tcp_snapshot > "$TMP/tcp_after"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "--- iperf3 summary ---"
|
||||
grep -E '\[SUM\].*(sender|receiver)' "$TMP/iperf.out" | tail -2
|
||||
echo ""
|
||||
echo "--- TCP SNMP deltas (/proc/net/snmp) ---"
|
||||
# Show only the counters that changed
|
||||
paste "$TMP/tcp_before" "$TMP/tcp_after" \
|
||||
| awk '{
|
||||
a=$3; b=$(NF)
|
||||
if (a+0 != b+0) printf " %-22s %12s -> %12s delta=%d\n", $1, a, b, b-a
|
||||
}'
|
||||
echo ""
|
||||
echo "--- softnet_stat deltas (look for non-zero dropped_delta/squeezed_delta) ---"
|
||||
softnet_delta "$TMP/softnet_before" "$TMP/softnet_after"
|
||||
echo ""
|
||||
echo "--- NIC counter deltas (only non-zero) ---"
|
||||
nic_delta "$TMP/nic_before" "$TMP/nic_after" "$HOST"
|
||||
echo ""
|
||||
echo "--- raw iperf3 tail (last 12 lines) ---"
|
||||
tail -12 "$TMP/iperf.out"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " HOW TO READ THIS"
|
||||
echo "==================================================================="
|
||||
echo " - If NIC rx_dropped / rx_missed_errors / rx_fifo_errors climbed:"
|
||||
echo " -> ring buffers too small. Fix: ethtool -G \$IF rx 4096 (or max)."
|
||||
echo " - If softnet_stat 'dropped' or 'squeezed' climbed on any CPU:"
|
||||
echo " -> softirq starvation. Fix: increase net.core.netdev_budget,"
|
||||
echo " check IRQ affinity, consider RPS."
|
||||
echo " - If NEITHER of the above climbed but TCP RetransSegs did:"
|
||||
echo " -> the retransmits are LACP reordering (out-of-order segments"
|
||||
echo " triggering fast retransmit), NOT packet loss. NOT fixable."
|
||||
echo "==================================================================="
|
||||
} | tee -a "$LOG"
|
||||
|
||||
echo ""
|
||||
echo "Log: $LOG"
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# lacp-rx-distribution.sh (HOST-NATIVE version)
|
||||
#
|
||||
# Runs ON a host (tsys6 or tsys7). Auto-detects role by hostname:
|
||||
# - On tsys6 (receiver): starts iperf3 -s -1 (one-shot), snapshots local
|
||||
# bond0 slave RX counters before+after, writes verdict + log.
|
||||
# - On tsys7 (sender): snapshots local bond0 slave TX counters before+
|
||||
# after, runs iperf3 -c <peer> -P 8 -t 12.
|
||||
#
|
||||
# Each side writes only its own counters. No inter-host SSH required.
|
||||
# iperf3 -s -1 (one-shot) handles client/server coordination.
|
||||
#
|
||||
# Output: /root/lacp-rx-{receiver,sender}.log
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
DURATION="${LACP_TEST_SECS:-12}"
|
||||
STREAMS="${LACP_STREAMS:-8}"
|
||||
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
||||
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
||||
LOG_DIR="/root"
|
||||
BOND="bond0"
|
||||
|
||||
# Snapshot per-slave byte counters from $BOND. Output: "<slave> rx tx\n" sorted.
|
||||
snapshot() {
|
||||
awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND" | while read -r s; do
|
||||
[ -n "$s" ] || continue
|
||||
rx=$(cat "/sys/class/net/$s/statistics/rx_bytes" 2>/dev/null || echo 0)
|
||||
tx=$(cat "/sys/class/net/$s/statistics/tx_bytes" 2>/dev/null || echo 0)
|
||||
printf '%s %s %s\n' "$s" "$rx" "$tx"
|
||||
done | sort
|
||||
}
|
||||
|
||||
# Compute per-slave deltas. $1=label, $2=col(2=rx,3=tx), $3=before, $4=after.
|
||||
analyze() {
|
||||
local label="$1" col="$2" before="$3" after="$4"
|
||||
join "$before" "$after" | awk -v col="$col" -v lbl="$label" '
|
||||
{
|
||||
slave=$1
|
||||
b = (col==2 ? $2 : $3)+0
|
||||
a = (col==2 ? $4 : $5)+0
|
||||
d = a - b; if (d < 0) d = 0
|
||||
delta[slave]=d; bef[slave]=b; aft[slave]=a
|
||||
order[++n]=slave; total+=d
|
||||
}
|
||||
END {
|
||||
printf "\n--- %s (bytes) ---\n", lbl
|
||||
printf " %-12s %14s %14s %14s %8s\n", "SLAVE", "BEFORE", "AFTER", "DELTA", "PCT"
|
||||
for (i=1;i<=n;i++){
|
||||
s=order[i]
|
||||
pct = (total>0 ? 100*delta[s]/total : 0)
|
||||
printf " %-12s %14d %14d %14d %7.1f%%\n", s, bef[s], aft[s], delta[s], pct
|
||||
}
|
||||
printf " %-12s %14s %14s %14d %8s\n", "TOTAL", "", "", total, "100.0%"
|
||||
}'
|
||||
}
|
||||
|
||||
# Dominant-slave pct for the verdict line. $1=col, $2=before, $3=after.
|
||||
dominant() {
|
||||
join "$2" "$3" | awk -v col="$1" '
|
||||
{ b=(col==2?$2:$3)+0; a=(col==2?$4:$5)+0; d=a-b; if(d<0)d=0; tot+=d; delta[$1]=d }
|
||||
END { m=0; ms=""; for (s in delta){ if (delta[s]>m){m=delta[s]; ms=s} }
|
||||
printf "%.1f %s", (tot>0?100*m/tot:0), ms }'
|
||||
}
|
||||
|
||||
HOST="$(uname -n)"
|
||||
HOST="${HOST%%.*}"
|
||||
case "$HOST" in
|
||||
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-rx-receiver.log" ;;
|
||||
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-rx-sender.log" ;;
|
||||
*) echo "ERROR: not running on tsys6 or tsys7 (hostname=$HOST)"; exit 2 ;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
{
|
||||
echo "=== LACP per-slave distribution ($ROLE) ==="
|
||||
echo "Host: $HOST Role: $ROLE"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Bond: $BOND"
|
||||
echo ""
|
||||
echo "--- $BOND state ---"
|
||||
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac|Slave Interface|Link" \
|
||||
"/proc/net/bonding/$BOND" 2>&1
|
||||
echo ""
|
||||
} | tee "$LOG"
|
||||
|
||||
snapshot > "$TMP/before"
|
||||
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
echo "[receiver] starting one-shot iperf3 server on $PEER_RX_IP ..."
|
||||
pkill -x iperf3 2>/dev/null; sleep 0.5
|
||||
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
||||
SRV_PID=$!
|
||||
# Wait for the server to be ready (brief), then wait for it to exit
|
||||
# (one-shot server exits after serving one client).
|
||||
for _ in $(seq 1 60); do
|
||||
kill -0 "$SRV_PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
wait "$SRV_PID" 2>/dev/null
|
||||
echo "[receiver] iperf3 server finished."
|
||||
else
|
||||
# sender: wait briefly for receiver to be listening, then run client.
|
||||
echo "[sender] waiting 3s for receiver to listen, then running iperf3 client..."
|
||||
sleep 3
|
||||
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
||||
> "$TMP/iperf.out" 2>&1
|
||||
echo "[sender] iperf3 client finished (exit=$?)."
|
||||
fi
|
||||
|
||||
snapshot > "$TMP/after"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "--- iperf3 output (raw tail) ---"
|
||||
tail -25 "$TMP/iperf.out"
|
||||
echo ""
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
analyze "Receiver RX (THE key column)" 2 "$TMP/before" "$TMP/after"
|
||||
dom="$(dominant 2 "$TMP/before" "$TMP/after")"
|
||||
pct=${dom%% *}; top=${dom##* }
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
|
||||
echo " VERDICT: SWITCH NOT distributing across $HOST's two ports"
|
||||
echo " Dominant slave '$top' = ${pct}% of RX -> hash not effective"
|
||||
echo " on this LAG. Action: verify/bounce pfv-r3-tor-stor port-channel."
|
||||
else
|
||||
echo " VERDICT: switch IS distributing (top slave '$top' = ${pct}%)."
|
||||
echo " Cap is host-side: softirq/CPU/bridge/NIC coalescing."
|
||||
fi
|
||||
echo "==================================================================="
|
||||
else
|
||||
analyze "Sender TX (control: should split if host hash=layer3+4)" 3 "$TMP/before" "$TMP/after"
|
||||
dom="$(dominant 3 "$TMP/before" "$TMP/after")"
|
||||
pct=${dom%% *}; top=${dom##* }
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
|
||||
echo " VERDICT: host TX NOT distributing (top slave '$top' = ${pct}%)."
|
||||
echo " Host xmit_hash_policy is NOT effective despite /proc/net/bonding."
|
||||
else
|
||||
echo " VERDICT: host TX distributing OK (top slave '$top' = ${pct}%)."
|
||||
fi
|
||||
echo "==================================================================="
|
||||
fi
|
||||
} | tee -a "$LOG"
|
||||
|
||||
echo ""
|
||||
echo "Log written: $LOG"
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/bin/bash
|
||||
# Read-only fleet drift probe — gathers package/tooling/config state for
|
||||
# consistency comparison across hosts. Writes only stdout.
|
||||
set -u
|
||||
echo "===== DRIFT PROBE: $(hostname -s) $(date -u +%FT%TZ) ====="
|
||||
echo
|
||||
echo "##### OS / kernel / PVE #####"
|
||||
pveversion 2>&1
|
||||
cat /etc/debian_version 2>&1
|
||||
echo
|
||||
echo "##### Installed key packages (versions) #####"
|
||||
dpkg-query -W -f='${Package}\t${Version}\n' 2>/dev/null | grep -iE 'lldpd|lldpad|smartmontools|nfs-common|nfs-kernel-server|iperf3|tcpdump|htop|rsyslog|qemu-guest-agent|snmpd|snmp|net-tools|ethtool|sysstat|ioping|fio|nvme-cli|conman|ser2net|nut-server|nut-client|tuned-adm|tuned' 2>/dev/null | sort
|
||||
echo
|
||||
echo "##### lldpcli present? #####"
|
||||
command -v lldpcli >/dev/null 2>&1 && lldpcli -v 2>&1 | head -1 || echo "lldpcli: NOT INSTALLED"
|
||||
echo
|
||||
echo "##### lldpd service #####"
|
||||
systemctl is-active lldpd 2>&1 || true
|
||||
systemctl is-enabled lldpd 2>&1 || true
|
||||
echo
|
||||
echo "##### smartmontools service #####"
|
||||
systemctl is-active smartd 2>&1 || echo "smartd: inactive/disabled"
|
||||
systemctl is-enabled smartd 2>&1 || true
|
||||
echo
|
||||
echo "##### snmpd service #####"
|
||||
systemctl is-active snmpd 2>&1 || echo "snmpd: not running"
|
||||
systemctl is-enabled snmpd 2>&1 || true
|
||||
echo
|
||||
echo "##### rsyslog #####"
|
||||
systemctl is-active rsyslog 2>&1 || echo "rsyslog: not running"
|
||||
echo
|
||||
echo "##### tuned profile #####"
|
||||
tuned-adm active 2>&1 | head -2 || echo "tuned: not available"
|
||||
echo
|
||||
echo "##### sshd config (key settings) #####"
|
||||
grep -E '^(PermitRootLogin|PasswordAuthentication|PubkeyAuthentication|KbdInteractiveAuthentication|Port |PermitEmptyPasswords)' /etc/ssh/sshd_config 2>/dev/null || echo "(defaults)"
|
||||
grep -E '^(PermitRootLogin|PasswordAuthentication|PubkeyAuthentication|KbdInteractiveAuthentication|Port |PermitEmptyPasswords)' /etc/ssh/sshd_config.d/*.conf 2>/dev/null || true
|
||||
echo
|
||||
echo "##### authorized_keys (root) — count + key types #####"
|
||||
if [ -f /root/.ssh/authorized_keys ]; then
|
||||
echo " total keys: $(wc -l < /root/.ssh/authorized_keys)"
|
||||
awk '{print " "$1}' /root/.ssh/authorized_keys | sort | uniq -c
|
||||
else
|
||||
echo " NO authorized_keys for root"
|
||||
fi
|
||||
echo
|
||||
echo "##### ssh host key fingerprints #####"
|
||||
for f in /etc/ssh/ssh_host_*_key.pub; do
|
||||
[ -f "$f" ] && ssh-keygen -lf "$f" 2>/dev/null
|
||||
done
|
||||
echo
|
||||
echo "##### sysctl tuning (key values) #####"
|
||||
echo " net.core.rmem_max=$(cat /proc/sys/net/core/rmem_max 2>/dev/null)"
|
||||
echo " net.core.wmem_max=$(cat /proc/sys/net/core/wmem_max 2>/dev/null)"
|
||||
echo " net.ipv4.tcp_congestion_control=$(cat /proc/sys/net/ipv4/tcp_congestion_control 2>/dev/null)"
|
||||
echo " net.ipv4.tcp_rmem=$(cat /proc/sys/net/ipv4/tcp_rmem 2>/dev/null)"
|
||||
echo " net.ipv4.tcp_wmem=$(cat /proc/sys/net/ipv4/tcp_wmem 2>/dev/null)"
|
||||
echo " net.ipv4.tcp_max_syn_backlog=$(cat /proc/sys/net/ipv4/tcp_max_syn_backlog 2>/dev/null)"
|
||||
echo " net.core.netdev_max_backlog=$(cat /proc/sys/net/core/netdev_max_backlog 2>/dev/null)"
|
||||
echo " vm.swappiness=$(cat /proc/sys/vm/swappiness 2>/dev/null)"
|
||||
echo
|
||||
echo "##### CPU governor #####"
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "no cpufreq driver"
|
||||
echo
|
||||
echo "##### beszel agent? #####"
|
||||
systemctl is-active beszel-hub 2>/dev/null || systemctl is-active beszel-agent 2>/dev/null || echo "beszel: not installed"
|
||||
echo
|
||||
echo "##### /root/bin or custom scripts present? #####"
|
||||
# shellcheck disable=SC2012 # diagnostic listing, not for processing
|
||||
find /root/bin/ -maxdepth 1 -type f 2>/dev/null | head -10 || echo "(no /root/bin)"
|
||||
find /root/ -maxdepth 1 -name '*.sh' -type f 2>/dev/null | head -10 || echo "(no /root/*.sh)"
|
||||
echo
|
||||
echo "##### mount options (noatime check on root) #####"
|
||||
mount | grep ' / ' | head -1
|
||||
echo
|
||||
echo "##### NFS server threads (if NFS server) #####"
|
||||
grep -E '^[[:space:]]*RPCNFSDCOUNT' /etc/default/nfs-kernel-server 2>/dev/null || grep -E 'RPCNFSDCOUNT' /etc/default/nfs-kernel-server 2>/dev/null || echo "(default, not a NFS server or default threads)"
|
||||
echo
|
||||
echo "===== END $(hostname -s) ====="
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# probe-network.sh
|
||||
#
|
||||
# READ-ONLY network + NFS ground-truth probe. Writes only stdout.
|
||||
# Run on any Proxmox host to inventory its NICs, bonds, LLDP neighbors,
|
||||
# NFS client mounts (including nconnect), ethtool link state, and error
|
||||
# counters. No hardcoded values — fully portable.
|
||||
#
|
||||
# Usage (via tests/remote.sh):
|
||||
# PROX_HOST=pfv-tsys6 bash tests/remote.sh prox-file perf/scripts/probe-network.sh
|
||||
###############################################################################
|
||||
set -u
|
||||
echo "===== HOST: $(hostname -s) $(date -u +%FT%TZ) ====="
|
||||
echo
|
||||
echo "##### ip -br link #####"
|
||||
ip -br link 2>&1
|
||||
echo
|
||||
echo "##### ip -br addr #####"
|
||||
ip -br addr 2>&1
|
||||
echo
|
||||
echo "##### /etc/network/interfaces #####"
|
||||
cat /etc/network/interfaces 2>&1
|
||||
echo
|
||||
echo "##### bond0 state (if present) #####"
|
||||
if [ -r /proc/net/bonding/bond0 ]; then
|
||||
cat /proc/net/bonding/bond0 2>&1
|
||||
else
|
||||
echo "(no bond0)"
|
||||
fi
|
||||
echo
|
||||
echo "##### ethtool per physical NIC #####"
|
||||
for nic in /sys/class/net/*; do
|
||||
nic=$(basename "$nic")
|
||||
case "$nic" in lo|bond*|br*|venet*|veth*|docker*|tap*|vnet*|fw*) continue;; esac
|
||||
echo "--- ethtool $nic ---"
|
||||
ethtool "$nic" 2>&1 | grep -iE 'Speed|Duplex|Port|Link|Supported link modes|Advertising|Auto-neg|Settings' || echo "(ethtool failed for $nic)"
|
||||
done
|
||||
echo
|
||||
echo "##### lldpcli (if installed) #####"
|
||||
if command -v lldpcli >/dev/null 2>&1; then
|
||||
echo "--- lldpcli show neighbors ---"
|
||||
lldpcli show neighbors 2>&1
|
||||
echo
|
||||
echo "--- lldpcli show interfaces ---"
|
||||
lldpcli show interfaces 2>&1
|
||||
echo
|
||||
echo "--- lldpcli show chassis ---"
|
||||
lldpcli show chassis 2>&1
|
||||
else
|
||||
echo "(lldpcli not installed)"
|
||||
fi
|
||||
echo
|
||||
echo "##### lldpd / lldpad service #####"
|
||||
systemctl is-active lldpd 2>&1 || true
|
||||
systemctl is-enabled lldpd 2>&1 || true
|
||||
echo
|
||||
echo "##### NFS mounts (mount | grep nfs) #####"
|
||||
mount | grep -i nfs 2>&1 || echo "(no nfs mounts)"
|
||||
echo
|
||||
echo "##### mount nconnect detail (nfsstat -m) #####"
|
||||
nfsstat -m 2>&1
|
||||
echo
|
||||
echo "##### storage.cfg NFS stanzas (options) #####"
|
||||
grep -A3 '^nfs:' /etc/pve/storage.cfg 2>&1
|
||||
echo
|
||||
echo "##### ip route #####"
|
||||
ip route 2>&1
|
||||
echo
|
||||
echo "##### ethtool -S bond slaves (key counters) #####"
|
||||
if [ -r /proc/net/bonding/bond0 ]; then
|
||||
# shellcheck disable=SC2013 # intentional: extract NIC names from bonding info
|
||||
for nic in $(grep -oE 'eth[0-9]+|en[psx][a-z0-9]+' /proc/net/bonding/bond0 2>/dev/null | sort -u); do
|
||||
echo "--- ethtool -S $nic (errors) ---"
|
||||
ethtool -S "$nic" 2>/dev/null | grep -iE 'error|drop|discard|crc|pause|miss' || echo "(no error counters)"
|
||||
done
|
||||
fi
|
||||
echo
|
||||
echo "##### ip neigh (ARP table, reachable/stale) #####"
|
||||
ip neigh show 2>&1 | grep -vE ' FAILED|INCOMPLETE' | sort -t. -k4 -n
|
||||
echo
|
||||
echo "===== END $(hostname -s) ====="
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# probe-storage.sh
|
||||
#
|
||||
# READ-ONLY storage + disk ground-truth probe. Writes only stdout.
|
||||
# Run on any Proxmox host (or any Linux NFS server) to inventory its physical
|
||||
# disks, mounts, exports, SMART health, and Proxmox storage config.
|
||||
#
|
||||
# Portable: no hardcoded values. Uses only standard CLI tools + smartmontools.
|
||||
#
|
||||
# Usage (via tests/remote.sh):
|
||||
# PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file perf/scripts/probe-storage.sh
|
||||
#
|
||||
# Or directly on a host:
|
||||
# bash probe-storage.sh > storage-audit.txt
|
||||
###############################################################################
|
||||
set -u
|
||||
echo "===== HOST: $(hostname -s) $(date -u +%FT%TZ) ====="
|
||||
echo
|
||||
echo "##### lsblk (tree, with model/serial/size/type) #####"
|
||||
lsblk -o NAME,MAJ:MIN,SIZE,TYPE,MOUNTPOINT,MODEL,SERIAL,STATE,ROTA,TRAN,REV 2>&1
|
||||
echo
|
||||
echo "##### block devices by-id #####"
|
||||
for dev in /dev/disk/by-id/*; do
|
||||
[ -L "$dev" ] || continue
|
||||
case "$(basename "$dev")" in *part[0-9]*) continue;; esac
|
||||
ls -l "$dev"
|
||||
done 2>&1
|
||||
echo
|
||||
echo "##### nvme list (if any) #####"
|
||||
command -v nvme >/dev/null 2>&1 && nvme list 2>&1 || echo "(no nvme-cli or no nvme devices)"
|
||||
echo
|
||||
echo "##### blkid #####"
|
||||
blkid 2>&1
|
||||
echo
|
||||
echo "##### mounted filesystems #####"
|
||||
findmnt -o TARGET,SOURCE,FSTYPE,OPTIONS 2>&1
|
||||
echo
|
||||
echo "##### /etc/fstab #####"
|
||||
cat /etc/fstab 2>&1
|
||||
echo
|
||||
echo "##### /etc/exports (+ exports.d) #####"
|
||||
cat /etc/exports 2>&1
|
||||
for f in /etc/exports.d/*.exports; do [ -f "$f" ] && echo "--- $f ---" && cat "$f"; done 2>&1
|
||||
echo
|
||||
echo "##### df -h (all mounts) #####"
|
||||
df -h 2>&1
|
||||
echo
|
||||
echo "##### smartctl -a per block device #####"
|
||||
command -v smartctl >/dev/null 2>&1 || echo "(smartctl not installed)"
|
||||
for d in /dev/sd? /dev/nvme?n1; do
|
||||
[ -b "$d" ] || continue
|
||||
echo "----- smartctl -a $d -----"
|
||||
smartctl -a "$d" 2>&1 | grep -iE 'Device Model|Model Number|Serial|Firmware|User Capacity|Rotation Rate|Form Factor|SATA Version|NVMe|SMART overall|Reallocated|Pending|Uncorrect|Power On|Temperature|Media and Data Integrity' || true
|
||||
done
|
||||
echo
|
||||
echo "##### /etc/pve/storage.cfg #####"
|
||||
cat /etc/pve/storage.cfg 2>&1
|
||||
echo
|
||||
echo "##### pvesm status #####"
|
||||
pvesm status 2>&1
|
||||
echo
|
||||
echo "##### pvesm list per store #####"
|
||||
for s in $(pvesm status 2>/dev/null | awk 'NR>1 && $3>0 {print $1}'); do
|
||||
echo "--- pvesm list $s ---"
|
||||
pvesm list "$s" 2>&1 | head -40
|
||||
done
|
||||
echo
|
||||
echo "##### zpool status (if any) #####"
|
||||
command -v zpool >/dev/null 2>&1 && zpool status 2>&1 || echo "(no zfs)"
|
||||
echo
|
||||
echo "##### lvm: pvs/vgs/lvs #####"
|
||||
command -v pvs >/dev/null 2>&1 && { pvs 2>&1; echo; vgs 2>&1; echo; lvs 2>&1; } || echo "(no lvm tools)"
|
||||
echo
|
||||
echo "===== END $(hostname -s) ====="
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# Read-only DNS consistency checker for Proxmox VMs.
|
||||
# Checks that every running VM's name resolves to the IP it's actually using,
|
||||
# and that reverse DNS (PTR) matches the VM name. Mismatches are reported as bugs.
|
||||
set -u
|
||||
echo "===== VM DNS CONSISTENCY CHECK: $(hostname -s) $(date -u +%FT%TZ) ====="
|
||||
echo
|
||||
echo "##### Running VMs with IPs (via guest-agent or ARP) #####"
|
||||
|
||||
for vmid in $(qm list 2>/dev/null | awk 'NR>1 && $4=="running" {print $1}'); do
|
||||
name=$(qm config "$vmid" 2>/dev/null | awk '/^name:/{print $2}')
|
||||
# Get the VM's net0 config to determine the bridge
|
||||
net0=$(qm config "$vmid" 2>/dev/null | awk '/^net0:/{print $0}')
|
||||
echo ""
|
||||
echo "--- VMID $vmid: $name ---"
|
||||
echo " net0: $(echo "$net0" | sed 's/net0: //')"
|
||||
|
||||
# Try to get IP via guest-agent
|
||||
if qm config "$vmid" 2>/dev/null | grep -q 'agent:.*enabled=1\|^agent: 1'; then
|
||||
guest_ips=$(qm guest cmd "$vmid" network-get-interfaces 2>/dev/null)
|
||||
if [ -n "$guest_ips" ]; then
|
||||
echo " guest-agent IPs:"
|
||||
echo "$guest_ips" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
for iface in data:
|
||||
name = iface.get('name','?')
|
||||
for addr in iface.get('ip-addresses',[]):
|
||||
ip = addr.get('ip-address','')
|
||||
typ = addr.get('ip-address-type','')
|
||||
if typ == 'ipv4' and not ip.startswith('127.'):
|
||||
print(f' {name}: {ip}')
|
||||
except: pass
|
||||
" 2>/dev/null
|
||||
else
|
||||
echo " guest-agent: no response"
|
||||
fi
|
||||
else
|
||||
echo " guest-agent: not enabled"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
echo "===== END $(hostname -s) ====="
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# run-lacp-retrans-cause.sh (workstation wrapper)
|
||||
#
|
||||
# Deploys scripts/lacp-retrans-cause.sh to BOTH hosts, runs them in the
|
||||
# right order, scps logs back, prints them.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
RECV="pfv-tsys6"; SEND="pfv-tsys7"
|
||||
LOCAL_LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/returned-logs/iperf"
|
||||
SCRIPT="lacp-retrans-cause.sh"
|
||||
LOCAL_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/scripts/${SCRIPT}"
|
||||
REMOTE_SCRIPT="/root/${SCRIPT}"
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " LACP retransmit-cause investigation"
|
||||
echo " Receiver: $RECV Sender: $SEND"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
echo "--- preflight ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
ssh "${SSH[@]}" "root@$h" \
|
||||
'command -v ethtool >/dev/null && e=OK || e=MISSING
|
||||
command -v iperf3 >/dev/null && i=OK || i=MISSING
|
||||
printf "ethtool=%s iperf3=%s host=%s\n" "$e" "$i" "$(uname -n)"' 2>&1 | head -1
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- deploy ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
||||
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
||||
&& echo "deployed" || echo "FAILED"
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- cleanup stale iperf3 ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- launching receiver on $RECV (background) ---"
|
||||
ssh "${SSH[@]}" "root@$RECV" \
|
||||
"nohup bash $REMOTE_SCRIPT > /root/lacp-retrans-receiver.console 2>&1 &" 2>/dev/null
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
||||
ssh "${SSH[@]}" "root@$SEND" "bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
||||
echo ""
|
||||
sleep 3
|
||||
|
||||
echo "--- fetching logs ---"
|
||||
for f in lacp-retrans-receiver.log lacp-retrans-receiver.console lacp-retrans-sender.log; do
|
||||
src=""
|
||||
case "$f" in
|
||||
*receiver*) src="$RECV" ;;
|
||||
*sender*) src="$SEND" ;;
|
||||
esac
|
||||
printf ' %-32s <- %s : ' "$f" "$src"
|
||||
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
||||
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
||||
else
|
||||
echo "MISSING"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "==================================================================="
|
||||
echo " RECEIVER LOG ($RECV)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-retrans-receiver.log" 2>/dev/null || echo "(missing)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SENDER LOG ($SEND)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-retrans-sender.log" 2>/dev/null || echo "(missing)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Local copies in: $LOCAL_LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# run-lacp-rx-distribution.sh (workstation wrapper)
|
||||
#
|
||||
# One-shot orchestrator: deploys scripts/lacp-rx-distribution.sh to BOTH
|
||||
# tsys6 (receiver) and tsys7 (sender), runs them in the right order, then
|
||||
# scps both logs back to returned-logs/iperf/.
|
||||
#
|
||||
# Run from the workstation:
|
||||
# bash scripts/run-lacp-rx-distribution.sh
|
||||
#
|
||||
# Idempotent: safe to re-run. iperf3 is killed on both hosts first.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
RECV="pfv-tsys6" # receiver (we care most about its RX split)
|
||||
SEND="pfv-tsys7" # sender (control: its TX split)
|
||||
LOCAL_LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/returned-logs/iperf"
|
||||
SCRIPT="lacp-rx-distribution.sh"
|
||||
LOCAL_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/scripts/${SCRIPT}"
|
||||
REMOTE_SCRIPT="/root/${SCRIPT}"
|
||||
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " LACP per-slave RX/TX distribution test"
|
||||
echo " Receiver: $RECV (10.100.100.6) Sender: $SEND (10.100.100.7)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# 0. Preflight: confirm SSH and iperf3 on both hosts
|
||||
echo "--- preflight (ssh + iperf3 + bond0) ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
ssh "${SSH[@]}" "root@$h" \
|
||||
'command -v iperf3 >/dev/null && ip=$(command -v iperf3) || ip=MISSING
|
||||
[ -r /proc/net/bonding/bond0 ] && b=OK || b=NO-BOND0
|
||||
printf "iperf3=%s bond0=%s host=%s\n" "$ip" "$b" "$(uname -n)"' \
|
||||
2>&1 | head -1
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 1. Copy the script to both hosts + chmod
|
||||
echo "--- deploy $SCRIPT to both hosts ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
||||
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
||||
&& echo "deployed + chmod +x" \
|
||||
|| echo "DEPLOY FAILED"
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 2. Kill any stale iperf3 on both hosts
|
||||
echo "--- cleanup stale iperf3 ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 3. Start RECEIVER in background (one-shot server, writes /root/lacp-rx-receiver.log)
|
||||
echo "--- starting receiver on $RECV (background) ---"
|
||||
ssh "${SSH[@]}" "root@$RECV" \
|
||||
"nohup bash $REMOTE_SCRIPT > /root/lacp-rx-receiver.console 2>&1 &" 2>/dev/null
|
||||
echo " receiver launched; waiting 5s for it to start iperf3 -s -1 ..."
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
# 4. Run SENDER (foreground; ~15s with the default 12s test + 3s pre-sleep)
|
||||
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
||||
ssh "${SSH[@]}" "root@$SEND" \
|
||||
"bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
||||
echo ""
|
||||
|
||||
# 5. Give receiver a moment to finish writing its log
|
||||
sleep 3
|
||||
|
||||
# 6. Fetch logs back
|
||||
echo "--- fetching logs ---"
|
||||
for f in lacp-rx-receiver.log lacp-rx-receiver.console lacp-rx-sender.log; do
|
||||
src=""
|
||||
case "$f" in
|
||||
lacp-rx-receiver*) src="$RECV" ;;
|
||||
lacp-rx-sender*) src="$SEND" ;;
|
||||
esac
|
||||
printf ' %-28s <- %s : ' "$f" "$src"
|
||||
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
||||
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
||||
else
|
||||
echo "MISSING"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 7. Show the receiver log (the decisive one)
|
||||
echo "==================================================================="
|
||||
echo " RECEIVER LOG ($RECV — the decisive side)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-rx-receiver.log" 2>/dev/null || echo "(no log)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SENDER LOG ($SEND — control)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-rx-sender.log" 2>/dev/null || echo "(no log)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Local copies in: $LOCAL_LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
snmp-switch-audit.py — READ-ONLY switch inventory via SNMP.
|
||||
|
||||
Gathers interface status/speed/errors, LLDP neighbor topology, port-channel
|
||||
(LAG) membership, and VLAN membership from any SNMPv2c-capable switch.
|
||||
Designed for Dell/Radlan (Neyland) and standard IF/LLDP/Q-BRIDGE MIB switches,
|
||||
but works on any SNMP-manageable device.
|
||||
|
||||
All parameters configurable via env vars or CLI flags so this works on any
|
||||
network:
|
||||
|
||||
SNMP_COMMUNITY SNMPv2c community string (env, default: public)
|
||||
SWITCH_IPS space-separated switch IPs (env, or pass as args)
|
||||
OUTPUT_DIR where to write per-switch (env, default: returned-logs/snmp)
|
||||
|
||||
Usage:
|
||||
SNMP_COMMUNITY=kn3lmgmt SWITCH_IPS="192.168.0.9 192.168.0.12" \\
|
||||
python3 snmp-switch-audit.py
|
||||
|
||||
# or pass IPs as positional args:
|
||||
SNMP_COMMUNITY=kn3lmgmt python3 snmp-switch-audit.py 192.168.0.9 192.168.0.12
|
||||
|
||||
Requires: pysnmp (pip install pysnmp) or net-snmp utils (snmpwalk) on PATH.
|
||||
Outputs: per-switch JSON + human-readable text in OUTPUT_DIR.
|
||||
|
||||
Read-only: sends only SNMP GET/GETNEXT/GETBULK. Never SETs anything.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
COMMUNITY = os.environ.get("SNMP_COMMUNITY", "public")
|
||||
OUTPUT_DIR = os.environ.get("OUTPUT_DIR",
|
||||
os.path.join(os.path.dirname(__file__), "..",
|
||||
"..", "returned-logs", "snmp"))
|
||||
|
||||
# OID constants
|
||||
OID_SYSDESCR = "1.3.6.1.2.1.1.1.0"
|
||||
OID_SYSNAME = "1.3.6.1.2.1.1.5.0"
|
||||
OID_IF_NAME = "1.3.6.1.2.1.31.1.1.1.1"
|
||||
OID_IF_SPEED = "1.3.6.1.2.1.2.2.1.5"
|
||||
OID_IF_OPER = "1.3.6.1.2.1.2.2.1.8"
|
||||
OID_IF_INERR = "1.3.6.1.2.1.2.2.1.14"
|
||||
OID_IF_OUTERR = "1.3.6.1.2.1.2.2.1.20"
|
||||
OID_IF_INOCT = "1.3.6.1.2.1.31.1.1.1.6"
|
||||
OID_IF_OUTOCT = "1.3.6.1.2.1.31.1.1.1.10"
|
||||
OID_LACP_LAG = "1.2.840.10006.300.43.1.1.1.1"
|
||||
OID_LLDP_REM_PORT = "1.0.8802.1.1.2.1.4.1.1.7"
|
||||
OID_LLDP_REM_SYSNAME = "1.0.8802.1.1.2.1.4.1.1.9"
|
||||
OID_LLDP_REM_CHASSIS = "1.0.8802.1.1.2.1.4.1.1.6"
|
||||
OID_LLDP_REM_LOCALPORT = "1.0.8802.1.1.2.1.4.1.1.3"
|
||||
OID_QBRIDGE_VLAN = "1.3.6.1.2.1.17.7.1.4.3.1.1"
|
||||
|
||||
|
||||
def snmpget(ip, oid):
|
||||
"""Single SNMP GET, returns string value or None."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["snmpget", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
if r.returncode == 0 and r.stdout.strip():
|
||||
return r.stdout.strip().strip('"')
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def snmpwalk(ip, oid):
|
||||
"""SNMP BULKWALK, returns dict of ifIndex -> value."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["snmpbulkwalk", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
if r.returncode != 0:
|
||||
r = subprocess.run(
|
||||
["snmpwalk", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return {}
|
||||
result = {}
|
||||
for i, line in enumerate(r.stdout.strip().split("\n"), 1):
|
||||
line = line.strip().strip('"')
|
||||
if line:
|
||||
result[i] = line
|
||||
return result
|
||||
|
||||
|
||||
def walk_indexed(ip, oid):
|
||||
"""SNMP walk preserving OID index. Returns dict: index_str -> value."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["snmpbulkwalk", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
if r.returncode != 0:
|
||||
r = subprocess.run(
|
||||
["snmpwalk", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return {}
|
||||
result = {}
|
||||
for line in r.stdout.strip().split("\n"):
|
||||
m = re.search(r'(\d+)\s*=\s*(.+)', line)
|
||||
if not m:
|
||||
m = re.search(r'\.(\d+)\s*=\s*(.+)', line)
|
||||
if m:
|
||||
idx = m.group(1).split(".")[-1]
|
||||
val = m.group(2).strip()
|
||||
val = re.sub(r'^(INTEGER: |STRING: |Hex-STRING: |Gauge32: |Counter32: |Counter64: )', '', val)
|
||||
result[idx] = val
|
||||
return result
|
||||
|
||||
|
||||
def audit_switch(ip):
|
||||
"""Gather all data for one switch."""
|
||||
data = {"ip": ip}
|
||||
data["sysDescr"] = snmpget(ip, OID_SYSDESCR)
|
||||
data["sysName"] = snmpget(ip, OID_SYSNAME)
|
||||
if not data["sysDescr"]:
|
||||
return data
|
||||
|
||||
names = snmpwalk(ip, OID_IF_NAME)
|
||||
speeds = snmpwalk(ip, OID_IF_SPEED)
|
||||
oper = snmpwalk(ip, OID_IF_OPER)
|
||||
inerr = snmpwalk(ip, OID_IF_INERR)
|
||||
outerr = snmpwalk(ip, OID_IF_OUTERR)
|
||||
|
||||
interfaces = []
|
||||
for idx in sorted(names.keys()):
|
||||
if idx not in names:
|
||||
continue
|
||||
speed_raw = speeds.get(idx, "0")
|
||||
try:
|
||||
speed_mbps = int(re.sub(r'\D', '', str(speed_raw))) // 1000000
|
||||
except (ValueError, TypeError):
|
||||
speed_mbps = 0
|
||||
is_up = str(oper.get(idx, "0")).strip() == "1"
|
||||
interfaces.append({
|
||||
"ifIndex": idx,
|
||||
"name": names[idx],
|
||||
"speedMbps": speed_mbps,
|
||||
"up": is_up,
|
||||
"inErrors": inerr.get(idx, "0"),
|
||||
"outErrors": outerr.get(idx, "0"),
|
||||
})
|
||||
data["interfaces"] = interfaces
|
||||
|
||||
# LLDP neighbors
|
||||
rem_ports = walk_indexed(ip, OID_LLDP_REM_PORT)
|
||||
rem_sysnames = walk_indexed(ip, OID_LLDP_REM_SYSNAME)
|
||||
rem_chassis = walk_indexed(ip, OID_LLDP_REM_CHASSIS)
|
||||
rem_local = walk_indexed(ip, OID_LLDP_REM_LOCALPORT)
|
||||
lldp = []
|
||||
for idx in rem_ports:
|
||||
lldp.append({
|
||||
"localPort": rem_local.get(idx, "?"),
|
||||
"remotePort": rem_ports[idx],
|
||||
"remoteSysName": rem_sysnames.get(idx, ""),
|
||||
"remoteChassis": rem_chassis.get(idx, ""),
|
||||
})
|
||||
data["lldpNeighbors"] = lldp
|
||||
|
||||
# LACP LAG table
|
||||
lag_data = walk_indexed(ip, OID_LACP_LAG)
|
||||
data["lagTable"] = lag_data
|
||||
|
||||
# VLAN membership
|
||||
vlan_data = walk_indexed(ip, OID_QBRIDGE_VLAN)
|
||||
data["vlans"] = vlan_data
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def print_switch(data):
|
||||
"""Human-readable summary."""
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {data.get('sysName', data['ip'])} ({data['ip']})")
|
||||
print(f" {data.get('sysDescr', '?')}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
print(f"\n Active ports (UP only):")
|
||||
print(f" {'Port':<12} {'Speed':>10} {'InErrors':>10} {'OutErrors':>10}")
|
||||
print(f" {'-'*12} {'-'*10} {'-'*10} {'-'*10}")
|
||||
for iface in data.get("interfaces", []):
|
||||
if iface["up"]:
|
||||
print(f" {iface['name']:<12} {iface['speedMbps']:>8}Mb "
|
||||
f"{iface['inErrors']:>10} {iface['outErrors']:>10}")
|
||||
|
||||
err_ports = [i for i in data.get("interfaces", [])
|
||||
if i["up"] and (int(i["inErrors"] or 0) > 0
|
||||
or int(i["outErrors"] or 0) > 0)]
|
||||
if err_ports:
|
||||
print(f"\n *** PORTS WITH ERRORS ***")
|
||||
for p in err_ports:
|
||||
print(f" {p['name']}: inErr={p['inErrors']} outErr={p['outErrors']}")
|
||||
|
||||
if data.get("lldpNeighbors"):
|
||||
print(f"\n LLDP neighbors:")
|
||||
for n in data["lldpNeighbors"]:
|
||||
sysname = n.get("remoteSysName", "") or "(unknown)"
|
||||
print(f" local={n['localPort']:<6} remote={n['remotePort']:<20} {sysname}")
|
||||
|
||||
if data.get("lagTable"):
|
||||
print(f"\n LACP/LAG table entries: {len(data['lagTable'])}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description="READ-ONLY SNMP switch audit (portable, config-driven)")
|
||||
ap.add_argument("switches", nargs="*",
|
||||
help="switch IPs (env: SWITCH_IPS)")
|
||||
ap.add_argument("--community", default=COMMUNITY,
|
||||
help=f"SNMPv2c community (env: SNMP_COMMUNITY, default: {COMMUNITY})")
|
||||
ap.add_argument("--output", default=OUTPUT_DIR,
|
||||
help=f"output dir (env: OUTPUT_DIR)")
|
||||
args = ap.parse_args()
|
||||
|
||||
community = args.community
|
||||
|
||||
ips = args.switches
|
||||
if not ips:
|
||||
env_ips = os.environ.get("SWITCH_IPS", "")
|
||||
ips = env_ips.split()
|
||||
|
||||
if not ips:
|
||||
ap.error("no switch IPs provided (pass as args or set SWITCH_IPS)")
|
||||
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
all_data = []
|
||||
|
||||
for ip in ips:
|
||||
globals()["COMMUNITY"] = community
|
||||
data = audit_switch(ip.strip())
|
||||
all_data.append(data)
|
||||
print_switch(data)
|
||||
outpath = os.path.join(args.output, f"switch-{ip}.json")
|
||||
with open(outpath, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
print(f"\n -> {outpath}")
|
||||
|
||||
combined = os.path.join(args.output, "switches-all.json")
|
||||
with open(combined, "w") as f:
|
||||
json.dump(all_data, f, indent=2)
|
||||
print(f"\n Combined: {combined}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user