chore: initialize repo with full project state
Performance optimization engagement for a 7-host Proxmox R&D cluster.
Captures the accumulated work across host tuning, network analysis,
fleet assessment, and kubernetes architecture planning.
Contents:
- Host-side tunings (scripts/): CPU governor, swappiness, BBR, NFS
nconnect, tuned profiles -- complete on 5 of 7 hosts
- Validation + benchmarking scripts: iperf matrix, bond/NFS fixes
- Collected host data (returned-logs/): check.sh output from all 7
hosts + iperf results, including newly-validated pfv-tsys9
- AGENTS.md: operating context for AI agents
- PROJECT.md: board-ready fleet assessment with VM placement and
storage redundancy analysis (40 VMs across 7 hosts)
- K8S.md: kubernetes architecture deep-dive covering cnode/wnode
distribution, StorageClass design, and ETL/HPC workload planning
💘 Generated with Crush
Assisted-by: Crush:glm-5.2
This commit is contained in:
Binary file not shown.
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# apply-bond-hash.sh
|
||||
#
|
||||
# Adds bond-xmit-hash-policy layer3+4 to bond0 in /etc/network/interfaces,
|
||||
# then reloads networking with ifreload -a.
|
||||
#
|
||||
# SSH survivability: this is safe IF your SSH session is on vmbr0/nic0
|
||||
# (management network), NOT on bond0/datanet (storage network).
|
||||
# tsys7's topology confirms this: SSH comes in on vmbr0 (nic0).
|
||||
#
|
||||
# Safety:
|
||||
# - Dry-run by default (--apply to commit)
|
||||
# - Full backup of /etc/network/interfaces
|
||||
# - Generates rollback script
|
||||
# - Does NOT reboot — uses ifreload -a which is hot-reload
|
||||
###############################################################################
|
||||
set -euo pipefail
|
||||
|
||||
HOST="$(hostname -s)"
|
||||
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
||||
ROLLBACK="/root/perfopt-bond-rollback-${TS_SHORT}.sh"
|
||||
ACTION="${1:-dryrun}"
|
||||
|
||||
[ "$ACTION" = "--apply" ] && ACTION="apply" || ACTION="dryrun"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " apply-bond-hash — $HOST"
|
||||
echo " mode: $ACTION"
|
||||
echo "==================================================================="
|
||||
|
||||
if [ ! -r /etc/network/interfaces ]; then
|
||||
echo "FATAL: /etc/network/interfaces not readable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if bond0 exists in the config
|
||||
if ! grep -q 'bond0' /etc/network/interfaces; then
|
||||
echo "No bond0 found in /etc/network/interfaces — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Check if the hash policy is already set
|
||||
if grep -q 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces; then
|
||||
echo "bond-xmit-hash-policy already present:"
|
||||
grep 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces
|
||||
echo "Checking value..."
|
||||
if grep -q 'layer3+4' /etc/network/interfaces; then
|
||||
echo "Already set to layer3+4 — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Show current bond state
|
||||
echo ""
|
||||
echo "--- Current bond0 state ---"
|
||||
cat /proc/net/bonding/bond0 2>/dev/null | head -15 || echo "(bond0 not up or not present)"
|
||||
|
||||
# Back up
|
||||
cp -a /etc/network/interfaces "$BACKUP_DIR/interfaces"
|
||||
|
||||
# Generate rollback script
|
||||
cat > "$ROLLBACK" <<EOF
|
||||
#!/bin/bash
|
||||
# Rollback for bond-xmit-hash-policy change
|
||||
# Restores original /etc/network/interfaces and reloads
|
||||
set -euo pipefail
|
||||
cp -a "$BACKUP_DIR/interfaces" /etc/network/interfaces
|
||||
echo "Restored /etc/network/interfaces"
|
||||
echo "Reloading networking..."
|
||||
ifreload -a 2>&1 || systemctl restart networking 2>&1 || true
|
||||
echo "Done. bond0 hash policy reverted to original."
|
||||
EOF
|
||||
chmod +x "$ROLLBACK"
|
||||
|
||||
echo ""
|
||||
echo "--- Proposed change ---"
|
||||
echo "Add line ' bond-xmit-hash-policy layer3+4' to the bond0 stanza."
|
||||
echo ""
|
||||
|
||||
if [ "$ACTION" != "apply" ]; then
|
||||
echo "DRY RUN — no changes made."
|
||||
echo "To commit: bash $0 --apply"
|
||||
echo "Rollback script (pre-generated): $ROLLBACK"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Apply: use sed to insert bond-xmit-hash-policy after bond-mode line
|
||||
# The bond0 stanza looks like:
|
||||
# auto bond0
|
||||
# iface bond0 inet manual
|
||||
# bond-slaves nic1 nic2
|
||||
# bond-miimon 100
|
||||
# bond-mode 802.3ad
|
||||
#
|
||||
# We insert after the bond-mode line.
|
||||
echo "Applying..."
|
||||
|
||||
# Check if bond-mode line exists (various formats)
|
||||
if grep -qE '^\s*bond-mode\s+802.3ad' /etc/network/interfaces; then
|
||||
# Insert after bond-mode 802.3ad line
|
||||
sed -i '/^\s*bond-mode\s+802\.3ad/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode line."
|
||||
elif grep -qE '^\s*bond-mode\s+4' /etc/network/interfaces; then
|
||||
sed -i '/^\s*bond-mode\s+4/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode 4 line."
|
||||
else
|
||||
echo "Could not find bond-mode line — inserting after bond-slaves line instead."
|
||||
sed -i '/^\s*bond-slaves/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
fi
|
||||
|
||||
# Show the result
|
||||
echo ""
|
||||
echo "--- Updated bond0 stanza ---"
|
||||
awk '/^auto bond0/,/^$/' /etc/network/interfaces
|
||||
|
||||
echo ""
|
||||
echo "--- Reloading networking (ifreload -a) ---"
|
||||
echo "SSH should survive (it's on vmbr0/nic0, not bond0)..."
|
||||
ifreload -a 2>&1 || {
|
||||
echo "ifreload failed, trying systemctl restart networking..."
|
||||
systemctl restart networking 2>&1
|
||||
}
|
||||
|
||||
# Wait a moment for bond to renegotiate
|
||||
echo "Waiting 5s for LACP to renegotiate..."
|
||||
sleep 5
|
||||
|
||||
echo ""
|
||||
echo "--- Post-change bond0 state ---"
|
||||
cat /proc/net/bonding/bond0 2>/dev/null | head -20
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " DONE."
|
||||
echo " Backup: $BACKUP_DIR/interfaces"
|
||||
echo " Rollback: bash $ROLLBACK"
|
||||
echo "==================================================================="
|
||||
Executable
+439
@@ -0,0 +1,439 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# apply-tunings.sh
|
||||
#
|
||||
# Applies the Tier 0 host-side tunings identified in FINDINGS.md:
|
||||
# 1. Sets scaling_governor=performance on every CPU
|
||||
# 2. Sets vm.swappiness appropriately (1 on storage hosts, 10 on VM hosts)
|
||||
# 3. Enables tcp_bbr module + sets congestion control
|
||||
# 4. Sets net.core.rmem_max/wmem_max + tcp_rmem/wmem for high-BDP NFS
|
||||
# 5. Sets the recommended tuned-adm profile (network-throughput or virtual-host)
|
||||
# 6. Adds nconnect=4 + noatime to NFS client mounts (edits /etc/pve/storage.cfg)
|
||||
# 7. (does NOT touch bond0 xmit-hash — that requires ifreload/network
|
||||
# restart which drops the host. Provided as a separate --emit-bond-patch
|
||||
# flag that PRINTS the change but does not apply it.)
|
||||
#
|
||||
# Safety features:
|
||||
# - Dry-run mode by default (--apply to commit)
|
||||
# - Full backup of every modified file to /root/perfopt-backup-<timestamp>/
|
||||
# - Per-host behaviour: detects storage hosts by hostname and applies the
|
||||
# right profile (tsys4, tsys5 = storage; others = VM hosts).
|
||||
# - Generates a /root/perfopt-rollback.sh that undoes everything.
|
||||
# - Does NOT reboot, does NOT restart networking, does NOT touch hardware.
|
||||
#
|
||||
# Usage:
|
||||
# bash apply-tunings.sh # dry run, show what would change
|
||||
# bash apply-tunings.sh --apply # commit changes
|
||||
# bash apply-tunings.sh --rollback # restore from latest backup
|
||||
# bash apply-tunings.sh --emit-bond-patch # show bond0 hash change (no apply)
|
||||
###############################################################################
|
||||
set -u
|
||||
umask 022
|
||||
|
||||
HOST="$(hostname -s)"
|
||||
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
||||
ROLLBACK_SCRIPT="/root/perfopt-rollback-${TS_SHORT}.sh"
|
||||
ACTION="dryrun"
|
||||
SKIP_NFS=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--apply) ACTION="apply" ;;
|
||||
--rollback) ACTION="rollback" ;;
|
||||
--emit-bond-patch) ACTION="bond-patch" ;;
|
||||
--no-nfs) SKIP_NFS=1 ;;
|
||||
-h|--help) sed -n '2,35p' "$0"; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Detect host role from hostname
|
||||
case "$HOST" in
|
||||
pfv-tsys4|pfv-tsys5)
|
||||
HOST_ROLE="storage"
|
||||
TUNED_PROFILE="network-throughput"
|
||||
SWAPPINESS="1"
|
||||
;;
|
||||
pfv-tsys1|pfv-tsys3|pfv-tsys6|pfv-tsys7)
|
||||
HOST_ROLE="vmhost"
|
||||
TUNED_PROFILE="virtual-host"
|
||||
SWAPPINESS="10"
|
||||
;;
|
||||
*)
|
||||
HOST_ROLE="unknown"
|
||||
TUNED_PROFILE="virtual-host"
|
||||
SWAPPINESS="10"
|
||||
;;
|
||||
esac
|
||||
|
||||
# --- rollback path ---------------------------------------------------------
|
||||
if [ "$ACTION" = "rollback" ]; then
|
||||
echo "Looking for latest backup under /root/perfopt-backup-* ..."
|
||||
latest=""
|
||||
while IFS= read -r d; do
|
||||
latest="$d"
|
||||
done < <(find /root -maxdepth 1 -type d -name 'perfopt-backup-*' 2>/dev/null | sort | tail -n1)
|
||||
if [ -z "$latest" ]; then
|
||||
echo "No backup found under /root/perfopt-backup-*" >&2
|
||||
exit 1
|
||||
fi
|
||||
rb="$latest/perfopt-rollback.sh"
|
||||
if [ ! -x "$rb" ] && [ ! -r "$rb" ]; then
|
||||
echo "Rollback script missing in $latest" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Rolling back using: $rb"
|
||||
bash "$rb"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# --- bond patch print-only path -------------------------------------------
|
||||
if [ "$ACTION" = "bond-patch" ]; then
|
||||
echo "==================================================================="
|
||||
echo " Proposed bond0 xmit_hash_policy change"
|
||||
echo "==================================================================="
|
||||
echo
|
||||
if [ ! -r /etc/network/interfaces ]; then
|
||||
echo "/etc/network/interfaces not readable"
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q 'bond-mode 802.3ad\|bond-mode 4\|bond-slaves' /etc/network/interfaces; then
|
||||
echo "No bond0 detected on this host — nothing to patch."
|
||||
exit 0
|
||||
fi
|
||||
cat <<EOF
|
||||
A manual edit to /etc/network/interfaces is required. The bond0 stanza
|
||||
needs this line added (it defaults to layer2, which is wrong for storage):
|
||||
|
||||
bond-xmit-hash-policy layer3+4
|
||||
|
||||
After editing, you MUST reload networking for it to take effect:
|
||||
ifreload -a # safe, brings interfaces down/up
|
||||
# OR
|
||||
systemctl restart networking # heavier, briefly drops connections
|
||||
|
||||
WARNING: applying this on a remote host over bond0 will briefly drop your
|
||||
SSH session. Run from console/IPMI, or schedule a maintenance window.
|
||||
|
||||
The change is reversible by removing the line and reloading again.
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- main path (dryrun or apply) ------------------------------------------
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Emit the rollback script header
|
||||
cat > "$ROLLBACK_SCRIPT" <<EOF
|
||||
#!/bin/bash
|
||||
# Auto-generated rollback for perfopt apply-tunings.sh
|
||||
# Backup timestamp: $TS
|
||||
# Backup dir: $BACKUP_DIR
|
||||
# Generated on: $HOST
|
||||
set -u
|
||||
EOF
|
||||
chmod +x "$ROLLBACK_SCRIPT"
|
||||
|
||||
backup_file() {
|
||||
local f="$1"
|
||||
if [ -e "$f" ]; then
|
||||
cp -a "$f" "$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
||||
fi
|
||||
}
|
||||
|
||||
# Helper: append a restore command to the rollback script
|
||||
rb_restore() {
|
||||
local f="$1"
|
||||
local bk
|
||||
bk="$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
if [ -r "$bk" ]; then
|
||||
cp -a "$bk" "$f" && echo "restored $f"
|
||||
else
|
||||
echo "WARN: backup $bk missing — $f left as-is"
|
||||
fi
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: append a sysctl restore command
|
||||
rb_sysctl_restore() {
|
||||
local key="$1" orig_val="$2"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
sysctl -w "$key=$orig_val" >/dev/null && echo "restored $key=$orig_val"
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: append a tuned-adm restore
|
||||
rb_tuned_restore() {
|
||||
local profile="$1"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
tuned-adm profile "$profile" 2>/dev/null && echo "restored tuned profile: $profile"
|
||||
EOF
|
||||
}
|
||||
|
||||
echo "==================================================================="
|
||||
echo " perfopt apply-tunings — $HOST"
|
||||
echo " mode: $ACTION"
|
||||
echo " role: $HOST_ROLE (tuned=$TUNED_PROFILE, swappiness=$SWAPPINESS)"
|
||||
echo " backup: $BACKUP_DIR"
|
||||
echo " rollback: $ROLLBACK_SCRIPT"
|
||||
echo "==================================================================="
|
||||
echo
|
||||
|
||||
# ===========================================================================
|
||||
# 1. scaling_governor → performance
|
||||
# ===========================================================================
|
||||
echo "--- 1. CPU scaling_governor → performance"
|
||||
if [ -r /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
|
||||
cur=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
|
||||
echo " current: $cur"
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
if [ "$cur" != "performance" ]; then
|
||||
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||
echo performance > "$c" 2>/dev/null || true
|
||||
done
|
||||
# Persist via systemd tmpfiles / sysctl fallback
|
||||
cat > /etc/systemd/system/perfopt-cpu-performance.service <<EOF
|
||||
[Unit]
|
||||
Description=Set CPU scaling_governor=performance (perfopt)
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/sh -c 'for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "\$c" 2>/dev/null || true; done'
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
systemctl enable perfopt-cpu-performance.service 2>/dev/null
|
||||
systemctl start perfopt-cpu-performance.service 2>/dev/null
|
||||
echo " applied + persisted via systemd unit"
|
||||
# Rollback
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "$cur" > "\$c" 2>/dev/null || true; done
|
||||
systemctl disable --now perfopt-cpu-performance.service 2>/dev/null || true
|
||||
rm -f /etc/systemd/system/perfopt-cpu-performance.service
|
||||
systemctl daemon-reload
|
||||
echo "restored scaling_governor=$cur (best-effort; original may have been dynamic)"
|
||||
EOF
|
||||
else
|
||||
echo " already performance — no change"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " cpufreq driver not loaded — nothing to do (typical on BIOS-locked servers)"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 2. vm.swappiness
|
||||
# ===========================================================================
|
||||
echo "--- 2. vm.swappiness → $SWAPPINESS (role=$HOST_ROLE)"
|
||||
cur_swappiness=$(sysctl -n vm.swappiness 2>/dev/null || echo "?")
|
||||
echo " current: $cur_swappiness"
|
||||
if [ "$ACTION" = "apply" ] && [ "$cur_swappiness" != "$SWAPPINESS" ]; then
|
||||
# Persist via sysctl.d
|
||||
backup_file /etc/sysctl.d/99-perfopt.conf
|
||||
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
||||
vm.swappiness = $SWAPPINESS
|
||||
EOF
|
||||
sysctl -w "vm.swappiness=$SWAPPINESS" >/dev/null
|
||||
echo " applied (persisted to /etc/sysctl.d/99-perfopt.conf)"
|
||||
rb_restore /etc/sysctl.d/99-perfopt.conf
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
rm -f /etc/sysctl.d/99-perfopt.conf
|
||||
sysctl -w "vm.swappiness=$cur_swappiness" >/dev/null
|
||||
echo "restored vm.swappiness=$cur_swappiness"
|
||||
EOF
|
||||
else
|
||||
echo " no change"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 3. TCP BBR + buffers (single sysctl.d file)
|
||||
# ===========================================================================
|
||||
echo "--- 3. TCP BBR + socket buffers"
|
||||
cur_cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo "?")
|
||||
cur_avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null || echo "?")
|
||||
echo " current: $cur_cc (available: $cur_avail)"
|
||||
|
||||
# Decide sysctl values (these are the consensus values for a 1-10 GbE NFS host)
|
||||
TARGET_RMEM_MAX=134217728 # 128 MB
|
||||
TARGET_WMEM_MAX=134217728
|
||||
TARGET_RMEM_DEFAULT=26214400 # 25 MB
|
||||
TARGET_WMEM_DEFAULT=26214400
|
||||
TARGET_TCP_RMEM="4096 87380 $TARGET_RMEM_MAX"
|
||||
TARGET_TCP_WMEM="4096 65536 $TARGET_RMEM_MAX"
|
||||
TARGET_NETDEV_MAX_BACKLOG=250000
|
||||
TARGET_SOMAXCONN=65535
|
||||
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
# Ensure tcp_bbr module loads at boot
|
||||
if ! grep -q '^tcp_bbr' /etc/modules-load.d/modules.conf 2>/dev/null; then
|
||||
backup_file /etc/modules-load.d/modules.conf
|
||||
mkdir -p /etc/modules-load.d
|
||||
cat >> /etc/modules-load.d/modules.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS
|
||||
tcp_bbr
|
||||
EOF
|
||||
modprobe tcp_bbr 2>/dev/null || true
|
||||
rb_restore /etc/modules-load.d/modules.conf
|
||||
fi
|
||||
|
||||
backup_file /etc/sysctl.d/99-perfopt.conf
|
||||
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
||||
vm.swappiness = $SWAPPINESS
|
||||
|
||||
# TCP BBR + high-BDP buffers (good for NFS over 1-10 GbE)
|
||||
net.ipv4.tcp_congestion_control = bbr
|
||||
net.core.default_qdisc = fq
|
||||
net.core.rmem_max = $TARGET_RMEM_MAX
|
||||
net.core.wmem_max = $TARGET_WMEM_MAX
|
||||
net.core.rmem_default = $TARGET_RMEM_DEFAULT
|
||||
net.core.wmem_default = $TARGET_WMEM_DEFAULT
|
||||
net.core.netdev_max_backlog = $TARGET_NETDEV_MAX_BACKLOG
|
||||
net.core.somaxconn = $TARGET_SOMAXCONN
|
||||
net.ipv4.tcp_rmem = $TARGET_TCP_RMEM
|
||||
net.ipv4.tcp_wmem = $TARGET_TCP_WMEM
|
||||
EOF
|
||||
sysctl --system >/dev/null 2>&1
|
||||
echo " applied (written to /etc/sysctl.d/99-perfopt.conf + sysctl --system)"
|
||||
# Rebuild rollback for sysctl
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
# Restore original sysctl state
|
||||
rm -f /etc/sysctl.d/99-perfopt.conf
|
||||
EOF
|
||||
rb_sysctl_restore "net.ipv4.tcp_congestion_control" "$cur_cc"
|
||||
rb_sysctl_restore "vm.swappiness" "$cur_swappiness"
|
||||
# Note: original buffer sizes not all captured; restoration is best-effort.
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
echo "Note: net buffers restored to kernel defaults (original values not captured)."
|
||||
echo "Run 'sysctl --system' to apply."
|
||||
sysctl --system >/dev/null 2>&1 || true
|
||||
EOF
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 4. tuned-adm profile
|
||||
# ===========================================================================
|
||||
echo "--- 4. tuned-adm profile → $TUNED_PROFILE"
|
||||
if command -v tuned-adm >/dev/null 2>&1; then
|
||||
cur_profile=$(tuned-adm active 2>/dev/null | awk -F: '/Current active profile/{gsub(/^[ \t]+/,"",$2); print $2}')
|
||||
echo " current: $cur_profile"
|
||||
if [ "$ACTION" = "apply" ] && [ "$cur_profile" != "$TUNED_PROFILE" ]; then
|
||||
tuned-adm profile "$TUNED_PROFILE" 2>&1 | sed 's/^/ /'
|
||||
rb_tuned_restore "$cur_profile"
|
||||
else
|
||||
echo " no change"
|
||||
fi
|
||||
else
|
||||
echo " tuned-adm not installed — skipping"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 5. NFS mount option hardening
|
||||
# Edit /etc/pve/storage.cfg to add nconnect=4 + noatime to NFS plugins.
|
||||
# Proxmox applies these on next mount/remount.
|
||||
# Skipped with --no-nfs (for NFS servers where we don't want to remount).
|
||||
# ===========================================================================
|
||||
echo "--- 5. NFS mount options: add nconnect=4, noatime"
|
||||
if [ "$SKIP_NFS" -eq 1 ]; then
|
||||
echo " SKIPPED (--no-nfs specified)"
|
||||
echo " Note: storage.cfg is cluster-wide; if edited on another host,"
|
||||
echo " the options are already staged here and activate on next reboot."
|
||||
else
|
||||
if [ -r /etc/pve/storage.cfg ]; then
|
||||
backup_file /etc/pve/storage.cfg
|
||||
if grep -q '^nfs:' /etc/pve/storage.cfg; then
|
||||
# Count how many nfs: stanzas lack the options
|
||||
while IFS= read -r line; do
|
||||
if echo "$line" | grep -q '^nfs:'; then
|
||||
cur_stanza=$(echo "$line" | awk '{print $2}')
|
||||
# Look ahead for the next few lines to see if options already set
|
||||
echo " nfs stanza: $cur_stanza"
|
||||
fi
|
||||
done < /etc/pve/storage.cfg
|
||||
|
||||
# Check whether any nfs stanza already has options
|
||||
if grep -A1 '^nfs:' /etc/pve/storage.cfg | grep -q 'options.*nconnect=4'; then
|
||||
echo " some stanzas already have nconnect=4 — check manually"
|
||||
else
|
||||
echo " proposed change: add 'options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2' to each nfs stanza"
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
# Apply via perl for safety (in-place edit with backup)
|
||||
if cp -a /etc/pve/storage.cfg "$BACKUP_DIR/etc__pve__storage.cfg"; then
|
||||
# Use a python helper for robust PVE storage.cfg editing
|
||||
python3 - <<PYEOF
|
||||
import re, pathlib
|
||||
p = pathlib.Path("/etc/pve/storage.cfg")
|
||||
text = p.read_text()
|
||||
pattern = re.compile(r'(^nfs:\s*\S+\n(?:[ \t]+[^\n]+\n)+)', re.MULTILINE)
|
||||
modified = 0
|
||||
def fix(m):
|
||||
global modified
|
||||
block = m.group(1)
|
||||
if 'options' in block:
|
||||
return block
|
||||
lines = block.rstrip('\n').split('\n')
|
||||
lines.insert(1, '\toptions nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2')
|
||||
modified += 1
|
||||
return '\n'.join(lines) + '\n'
|
||||
new_text = pattern.sub(fix, text)
|
||||
if modified:
|
||||
p.write_text(new_text)
|
||||
print(f" patched {modified} NFS stanzas with options line")
|
||||
else:
|
||||
print(" no NFS stanza needed patching (all already had options or no nfs: found)")
|
||||
PYEOF
|
||||
# Remount existing NFS mounts to pick up new options
|
||||
echo " remounting NFS mounts to apply new options..."
|
||||
while IFS= read -r mp; do
|
||||
mount -o remount "$mp" 2>/dev/null && echo " remounted $mp" || echo " FAILED to remount $mp (will pick up on next mount)"
|
||||
done < <(awk '$3 ~ /^nfs/{print $2}' /proc/mounts 2>/dev/null | sort -u)
|
||||
rb_restore /etc/pve/storage.cfg
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
# To fully undo NFS options, also remount:
|
||||
while IFS= read -r mp; do
|
||||
mount -o remount "\$mp" 2>/dev/null
|
||||
done < <(awk '\$3 ~ /^nfs/{print \$2}' /proc/mounts 2>/dev/null | sort -u)
|
||||
echo "restored NFS mount options"
|
||||
EOF
|
||||
else
|
||||
echo " ERROR: could not back up storage.cfg — aborting NFS patch"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " no NFS stanzas in storage.cfg — skipping"
|
||||
fi
|
||||
else
|
||||
echo " /etc/pve/storage.cfg not readable — skipping"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 6. Summary
|
||||
# ===========================================================================
|
||||
echo
|
||||
echo "==================================================================="
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
echo " DONE. Backups in: $BACKUP_DIR"
|
||||
echo " Rollback: bash $ROLLBACK_SCRIPT"
|
||||
echo
|
||||
echo " Next steps:"
|
||||
echo " - Verify with 'sysctl net.ipv4.tcp_congestion_control vm.swappiness'"
|
||||
echo " - Verify NFS mounts with 'nfsstat -m' (look for nconnect=4)"
|
||||
echo " - Run 'bash apply-tunings.sh --emit-bond-patch' for the bond0"
|
||||
echo " xmit_hash_policy change (requires network restart, do in"
|
||||
echo " maintenance window)"
|
||||
else
|
||||
echo " DRY RUN — no changes made."
|
||||
echo " To commit: bash apply-tunings.sh --apply"
|
||||
fi
|
||||
echo "==================================================================="
|
||||
Executable
+1138
File diff suppressed because it is too large
Load Diff
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
# fix-tsys7.sh - fixes the two issues found: NFS options string + bond hash
|
||||
# Runs on the target host directly.
|
||||
set -euo pipefail
|
||||
|
||||
echo "==================================================================="
|
||||
echo " FIX: NFS options + bond hash policy"
|
||||
echo "==================================================================="
|
||||
|
||||
# --- FIX 1: Simplify NFS options in storage.cfg ---------------------------
|
||||
echo ""
|
||||
echo "--- FIX 1: Simplify NFS options (remove version=4.2 conflict) ---"
|
||||
# Replace the overly-complex options line with a minimal one
|
||||
# PVE handles vers/rsize/wsize/hard/etc internally; we only need nconnect + noatime
|
||||
if grep -q 'options.*nconnect=4.*version=4.2' /etc/pve/storage.cfg; then
|
||||
# Use sed to replace each options line
|
||||
sed -i 's/options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2/options nconnect=4,noatime/' /etc/pve/storage.cfg
|
||||
echo "Fixed: simplified NFS options to 'nconnect=4,noatime'"
|
||||
else
|
||||
echo "Options line already simplified or not present"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Verify storage.cfg NFS stanzas ---"
|
||||
grep -A 2 "^nfs: D2" /etc/pve/storage.cfg | head -3
|
||||
echo "..."
|
||||
grep -c "options nconnect" /etc/pve/storage.cfg
|
||||
echo " NFS stanzas with options"
|
||||
|
||||
# --- FIX 2: Bond hash policy (force via sysfs + persist in interfaces) ----
|
||||
echo ""
|
||||
echo "--- FIX 2: Apply bond xmit_hash_policy=layer3+4 ---"
|
||||
|
||||
# 2a: Apply LIVE via sysfs (takes effect immediately, no network reload)
|
||||
echo "Applying live via sysfs..."
|
||||
if echo "layer3+4" > /sys/class/net/bond0/bonding/xmit_hash_policy 2>/dev/null; then
|
||||
echo "Live sysfs apply: SUCCESS"
|
||||
else
|
||||
echo "Live sysfs apply: FAILED (will persist in config and apply on ifreload)"
|
||||
fi
|
||||
|
||||
# Verify live state
|
||||
echo ""
|
||||
echo "Live bond0 hash policy:"
|
||||
cat /proc/net/bonding/bond0 | grep "Transmit Hash"
|
||||
|
||||
# 2b: Persist in /etc/network/interfaces (fix the sed that failed before)
|
||||
echo ""
|
||||
echo "Persisting in /etc/network/interfaces..."
|
||||
|
||||
# Backup
|
||||
cp -a /etc/network/interfaces "/root/interfaces.bondfix.$(date +%Y%m%d%H%M%S)"
|
||||
|
||||
# Check if already present
|
||||
if grep -q 'bond-xmit-hash-policy' /etc/network/interfaces; then
|
||||
echo "bond-xmit-hash-policy already in interfaces file"
|
||||
else
|
||||
# Use awk to insert after the bond-mode line (more reliable than sed)
|
||||
# Match any line containing 'bond-mode' (regardless of indentation)
|
||||
awk '
|
||||
/bond-mode/ && !done {
|
||||
print
|
||||
print "\tbond-xmit-hash-policy layer3+4"
|
||||
done=1
|
||||
next
|
||||
}
|
||||
{ print }
|
||||
' /etc/network/interfaces > /etc/network/interfaces.new
|
||||
|
||||
mv /etc/network/interfaces.new /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Updated bond0 stanza ---"
|
||||
awk '/^auto bond0/,/^$/' /etc/network/interfaces
|
||||
|
||||
# Final verify
|
||||
echo ""
|
||||
echo "--- Final bond0 running state ---"
|
||||
cat /proc/net/bonding/bond0 | head -20
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " FIX COMPLETE"
|
||||
echo "==================================================================="
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# lacp-retrans-cause.sh (HOST-NATIVE)
|
||||
#
|
||||
# Runs on tsys6 (receiver) or tsys7 (sender). Auto-detects role.
|
||||
#
|
||||
# Question this answers: are the ~56K retransmits we see on 8-stream iperf
|
||||
# between tsys6 and tsys7 fixable (NIC ring drops, softnet drops, CPU
|
||||
# saturation) or just unavoidable LACP reordering overhead?
|
||||
#
|
||||
# Method: snapshot drop/error counters + softnet_stat + TCP SNMP before and
|
||||
# after a 12s iperf3 run, then print the deltas. The decisive columns are:
|
||||
# - NIC rx_dropped / rx_missed_errors / rx_no_dma_resources -> ring too small
|
||||
# - /proc/net/softnet_stat drops -> ksoftirq starved
|
||||
# - /proc/net/snmp TCP retranst -> TCP-level retrans
|
||||
# If NIC drop counters do NOT climb but TCP retrans does, the retrans are
|
||||
# coming from LACP reordering (out-of-order segments triggering fast
|
||||
# retransmit), not packet loss — and are NOT fixable by tuning.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
DURATION="${LACP_TEST_SECS:-12}"
|
||||
STREAMS="${LACP_STREAMS:-8}"
|
||||
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
||||
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
||||
LOG_DIR="/root"
|
||||
BOND="bond0"
|
||||
|
||||
HOST="$(uname -n)"; HOST="${HOST%%.*}"
|
||||
case "$HOST" in
|
||||
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-retrans-receiver.log" ;;
|
||||
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-retrans-sender.log" ;;
|
||||
*) echo "ERROR: not on tsys6 or tsys7 (host=$HOST)"; exit 2 ;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# ---- per-slave NIC counter snapshot -----------------------------------------
|
||||
# Captures rx_dropped, rx_missed_errors, rx_no_dma_resources, tx_retrans,
|
||||
# plus all error-like counters. Falls back gracefully if a counter doesn't
|
||||
# exist (different NIC drivers expose different names).
|
||||
nic_snapshot() {
|
||||
for s in $(awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND"); do
|
||||
[ -n "$s" ] || continue
|
||||
echo "[$s]"
|
||||
# ethtool -S may exist; show only drop/error/retrans lines
|
||||
if command -v ethtool >/dev/null 2>&1; then
|
||||
ethtool -S "$s" 2>/dev/null \
|
||||
| grep -iE 'drop|miss|error|no_dma|fifo|retrans|overflow' \
|
||||
|| echo "(ethtool -S: no matching counters or unsupported)"
|
||||
else
|
||||
echo "(ethtool not installed)"
|
||||
fi
|
||||
# /sys counters (always available, driver-agnostic)
|
||||
for c in rx_dropped tx_dropped rx_errors tx_errors rx_missed_errors \
|
||||
rx_length_errors rx_crc_errors rx_fifo_errors tx_fifo_errors \
|
||||
multicast collisions; do
|
||||
v=$(cat "/sys/class/net/$s/statistics/$c" 2>/dev/null)
|
||||
[ -n "$v" ] && printf ' sysfs %-22s = %s\n' "$c" "$v"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# ---- softnet_stat snapshot (per-CPU RX softirq drops) -----------------------
|
||||
# /proc/net/softnet_stat columns are HEX. Col1=processed, col2=dropped,
|
||||
# col3=time_squeeze. We only care about dropped + squeeze (small numbers that
|
||||
# look identical in hex and decimal). Processed is informational only and we
|
||||
# don't try to delta it (hex arithmetic is awk-version-dependent).
|
||||
softnet_snapshot() {
|
||||
awk '{ printf "cpu%s processed=%s dropped=%s squeezed=%s\n", \
|
||||
NR-1, $1, $2, $3 }' /proc/net/softnet_stat
|
||||
}
|
||||
|
||||
# ---- TCP SNMP snapshot (the source of iperf's retransmit number) -----------
|
||||
tcp_snapshot() {
|
||||
awk '/^Tcp:/{
|
||||
if (!seen) {
|
||||
seen=1
|
||||
# /proc/net/snmp Tcp line 1 = names, line 2 = values
|
||||
n=split($0, names, " ")
|
||||
# re-find the values line
|
||||
getline
|
||||
vals=$0
|
||||
split(vals, v, " ")
|
||||
for (i=1;i<=n;i++) printf " %-22s = %s\n", names[i], v[i]
|
||||
}
|
||||
}' /proc/net/snmp
|
||||
}
|
||||
|
||||
# ---- print deltas for selected counters ------------------------------------
|
||||
# $1 = before file, $2 = after file
|
||||
# softnet_snapshot output: "cpuN processed=HEX dropped=HEX squeezed=HEX"
|
||||
# /proc/net/softnet_stat is hex, but dropped/squeezed are always small
|
||||
# integers (typically 0 on a healthy host), so +0 coercion is correct for
|
||||
# the values we care about. We deliberately DON'T report `processed` deltas
|
||||
# because hex arithmetic on large numbers is awk-version-dependent.
|
||||
softnet_delta() {
|
||||
awk '
|
||||
FNR==NR { if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
||||
split($0, p, /[= ]+/); drop[id]=p[5]+0; sqz[id]=p[7]+0 }
|
||||
next }
|
||||
{ if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
||||
split($0, q, /[= ]+/);
|
||||
dd = (q[5]+0) - drop[id]
|
||||
sd = (q[7]+0) - sqz[id]
|
||||
if (dd != 0 || sd != 0)
|
||||
printf " %-8s dropped_delta=%-6s squeezed_delta=%-6s\n", id, dd, sd } }
|
||||
' "$1" "$2" | sort
|
||||
}
|
||||
|
||||
# $1 = before file, $2 = after file, $3 = section label prefix
|
||||
# File format: slave header line "[nic1]", then " counter_name = value" lines.
|
||||
# Output only counters whose value changed.
|
||||
nic_delta() {
|
||||
awk '
|
||||
FNR==NR { if ($0 ~ /^\[/) slave=$0
|
||||
else if ($0 ~ /=/) {
|
||||
# Everything before " = " is the counter name (trim spaces)
|
||||
pos = index($0, "=")
|
||||
name = substr($0, 1, pos-1)
|
||||
gsub(/^ +| +$/, "", name)
|
||||
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
||||
before[slave SUBSEP name] = val
|
||||
}
|
||||
next }
|
||||
{ if ($0 ~ /^\[/) slave=$0
|
||||
else if ($0 ~ /=/) {
|
||||
pos = index($0, "=")
|
||||
name = substr($0, 1, pos-1); gsub(/^ +| +$/, "", name)
|
||||
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
||||
b = before[slave SUBSEP name]+0
|
||||
d = val+0 - b
|
||||
if (d != 0) printf " %-8s %-30s %12s -> %12s delta=%d\n", slave, name, b, val, d
|
||||
} }
|
||||
' "$1" "$2"
|
||||
}
|
||||
|
||||
{
|
||||
echo "=== LACP retransmit cause investigation ($ROLE) ==="
|
||||
echo "Host: $HOST Role: $ROLE"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Bond: $BOND"
|
||||
echo ""
|
||||
echo "--- bond0 hash + driver ---"
|
||||
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac" /proc/net/bonding/$BOND
|
||||
for s in $(awk '/^Slave Interface:/{print $3}' /proc/net/bonding/$BOND); do
|
||||
drv=$(ethtool -i "$s" 2>/dev/null | awk -F: '/^driver:/{print $2}' | sed 's/^ *//')
|
||||
speed=$(cat /sys/class/net/$s/speed 2>/dev/null)
|
||||
ring=$(ethtool -g "$s" 2>/dev/null | awk '/RX:/{print $2; exit}')
|
||||
printf " %-8s driver=%-20s speed=%-6s current RX ring=%s\n" "$s" "$drv" "$speed" "$ring"
|
||||
done
|
||||
echo ""
|
||||
} | tee "$LOG"
|
||||
|
||||
echo "--- BEFORE snapshots ---"
|
||||
nic_snapshot > "$TMP/nic_before"
|
||||
softnet_snapshot > "$TMP/softnet_before"
|
||||
tcp_snapshot > "$TMP/tcp_before"
|
||||
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
echo "[receiver] starting iperf3 -s -1 on $PEER_RX_IP ..."
|
||||
pkill -x iperf3 2>/dev/null; sleep 0.5
|
||||
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
||||
SRV_PID=$!
|
||||
for _ in $(seq 1 60); do kill -0 "$SRV_PID" 2>/dev/null || break; sleep 1; done
|
||||
wait "$SRV_PID" 2>/dev/null
|
||||
else
|
||||
echo "[sender] waiting 3s for receiver to listen, then running iperf3 -P $STREAMS -t $DURATION ..."
|
||||
sleep 3
|
||||
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
||||
> "$TMP/iperf.out" 2>&1
|
||||
fi
|
||||
|
||||
echo "--- AFTER snapshots ---"
|
||||
nic_snapshot > "$TMP/nic_after"
|
||||
softnet_snapshot > "$TMP/softnet_after"
|
||||
tcp_snapshot > "$TMP/tcp_after"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "--- iperf3 summary ---"
|
||||
grep -E '\[SUM\].*(sender|receiver)' "$TMP/iperf.out" | tail -2
|
||||
echo ""
|
||||
echo "--- TCP SNMP deltas (/proc/net/snmp) ---"
|
||||
# Show only the counters that changed
|
||||
paste "$TMP/tcp_before" "$TMP/tcp_after" \
|
||||
| awk '{
|
||||
a=$3; b=$(NF)
|
||||
if (a+0 != b+0) printf " %-22s %12s -> %12s delta=%d\n", $1, a, b, b-a
|
||||
}'
|
||||
echo ""
|
||||
echo "--- softnet_stat deltas (look for non-zero dropped_delta/squeezed_delta) ---"
|
||||
softnet_delta "$TMP/softnet_before" "$TMP/softnet_after"
|
||||
echo ""
|
||||
echo "--- NIC counter deltas (only non-zero) ---"
|
||||
nic_delta "$TMP/nic_before" "$TMP/nic_after" "$HOST"
|
||||
echo ""
|
||||
echo "--- raw iperf3 tail (last 12 lines) ---"
|
||||
tail -12 "$TMP/iperf.out"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " HOW TO READ THIS"
|
||||
echo "==================================================================="
|
||||
echo " - If NIC rx_dropped / rx_missed_errors / rx_fifo_errors climbed:"
|
||||
echo " -> ring buffers too small. Fix: ethtool -G \$IF rx 4096 (or max)."
|
||||
echo " - If softnet_stat 'dropped' or 'squeezed' climbed on any CPU:"
|
||||
echo " -> softirq starvation. Fix: increase net.core.netdev_budget,"
|
||||
echo " check IRQ affinity, consider RPS."
|
||||
echo " - If NEITHER of the above climbed but TCP RetransSegs did:"
|
||||
echo " -> the retransmits are LACP reordering (out-of-order segments"
|
||||
echo " triggering fast retransmit), NOT packet loss. NOT fixable."
|
||||
echo "==================================================================="
|
||||
} | tee -a "$LOG"
|
||||
|
||||
echo ""
|
||||
echo "Log: $LOG"
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# lacp-rx-distribution.sh (HOST-NATIVE version)
|
||||
#
|
||||
# Runs ON a host (tsys6 or tsys7). Auto-detects role by hostname:
|
||||
# - On tsys6 (receiver): starts iperf3 -s -1 (one-shot), snapshots local
|
||||
# bond0 slave RX counters before+after, writes verdict + log.
|
||||
# - On tsys7 (sender): snapshots local bond0 slave TX counters before+
|
||||
# after, runs iperf3 -c <peer> -P 8 -t 12.
|
||||
#
|
||||
# Each side writes only its own counters. No inter-host SSH required.
|
||||
# iperf3 -s -1 (one-shot) handles client/server coordination.
|
||||
#
|
||||
# Output: /root/lacp-rx-{receiver,sender}.log
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
DURATION="${LACP_TEST_SECS:-12}"
|
||||
STREAMS="${LACP_STREAMS:-8}"
|
||||
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
||||
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
||||
LOG_DIR="/root"
|
||||
BOND="bond0"
|
||||
|
||||
# Snapshot per-slave byte counters from $BOND. Output: "<slave> rx tx\n" sorted.
|
||||
snapshot() {
|
||||
awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND" | while read -r s; do
|
||||
[ -n "$s" ] || continue
|
||||
rx=$(cat "/sys/class/net/$s/statistics/rx_bytes" 2>/dev/null || echo 0)
|
||||
tx=$(cat "/sys/class/net/$s/statistics/tx_bytes" 2>/dev/null || echo 0)
|
||||
printf '%s %s %s\n' "$s" "$rx" "$tx"
|
||||
done | sort
|
||||
}
|
||||
|
||||
# Compute per-slave deltas. $1=label, $2=col(2=rx,3=tx), $3=before, $4=after.
|
||||
analyze() {
|
||||
local label="$1" col="$2" before="$3" after="$4"
|
||||
join "$before" "$after" | awk -v col="$col" -v lbl="$label" '
|
||||
{
|
||||
slave=$1
|
||||
b = (col==2 ? $2 : $3)+0
|
||||
a = (col==2 ? $4 : $5)+0
|
||||
d = a - b; if (d < 0) d = 0
|
||||
delta[slave]=d; bef[slave]=b; aft[slave]=a
|
||||
order[++n]=slave; total+=d
|
||||
}
|
||||
END {
|
||||
printf "\n--- %s (bytes) ---\n", lbl
|
||||
printf " %-12s %14s %14s %14s %8s\n", "SLAVE", "BEFORE", "AFTER", "DELTA", "PCT"
|
||||
for (i=1;i<=n;i++){
|
||||
s=order[i]
|
||||
pct = (total>0 ? 100*delta[s]/total : 0)
|
||||
printf " %-12s %14d %14d %14d %7.1f%%\n", s, bef[s], aft[s], delta[s], pct
|
||||
}
|
||||
printf " %-12s %14s %14s %14d %8s\n", "TOTAL", "", "", total, "100.0%"
|
||||
}'
|
||||
}
|
||||
|
||||
# Dominant-slave pct for the verdict line. $1=col, $2=before, $3=after.
|
||||
dominant() {
|
||||
join "$2" "$3" | awk -v col="$1" '
|
||||
{ b=(col==2?$2:$3)+0; a=(col==2?$4:$5)+0; d=a-b; if(d<0)d=0; tot+=d; delta[$1]=d }
|
||||
END { m=0; ms=""; for (s in delta){ if (delta[s]>m){m=delta[s]; ms=s} }
|
||||
printf "%.1f %s", (tot>0?100*m/tot:0), ms }'
|
||||
}
|
||||
|
||||
HOST="$(uname -n)"
|
||||
HOST="${HOST%%.*}"
|
||||
case "$HOST" in
|
||||
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-rx-receiver.log" ;;
|
||||
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-rx-sender.log" ;;
|
||||
*) echo "ERROR: not running on tsys6 or tsys7 (hostname=$HOST)"; exit 2 ;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
{
|
||||
echo "=== LACP per-slave distribution ($ROLE) ==="
|
||||
echo "Host: $HOST Role: $ROLE"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Bond: $BOND"
|
||||
echo ""
|
||||
echo "--- $BOND state ---"
|
||||
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac|Slave Interface|Link" \
|
||||
"/proc/net/bonding/$BOND" 2>&1
|
||||
echo ""
|
||||
} | tee "$LOG"
|
||||
|
||||
snapshot > "$TMP/before"
|
||||
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
echo "[receiver] starting one-shot iperf3 server on $PEER_RX_IP ..."
|
||||
pkill -x iperf3 2>/dev/null; sleep 0.5
|
||||
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
||||
SRV_PID=$!
|
||||
# Wait for the server to be ready (brief), then wait for it to exit
|
||||
# (one-shot server exits after serving one client).
|
||||
for _ in $(seq 1 60); do
|
||||
kill -0 "$SRV_PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
wait "$SRV_PID" 2>/dev/null
|
||||
echo "[receiver] iperf3 server finished."
|
||||
else
|
||||
# sender: wait briefly for receiver to be listening, then run client.
|
||||
echo "[sender] waiting 3s for receiver to listen, then running iperf3 client..."
|
||||
sleep 3
|
||||
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
||||
> "$TMP/iperf.out" 2>&1
|
||||
echo "[sender] iperf3 client finished (exit=$?)."
|
||||
fi
|
||||
|
||||
snapshot > "$TMP/after"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "--- iperf3 output (raw tail) ---"
|
||||
tail -25 "$TMP/iperf.out"
|
||||
echo ""
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
analyze "Receiver RX (THE key column)" 2 "$TMP/before" "$TMP/after"
|
||||
dom="$(dominant 2 "$TMP/before" "$TMP/after")"
|
||||
pct=${dom%% *}; top=${dom##* }
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
|
||||
echo " VERDICT: SWITCH NOT distributing across $HOST's two ports"
|
||||
echo " Dominant slave '$top' = ${pct}% of RX -> hash not effective"
|
||||
echo " on this LAG. Action: verify/bounce pfv-r3-tor-stor port-channel."
|
||||
else
|
||||
echo " VERDICT: switch IS distributing (top slave '$top' = ${pct}%)."
|
||||
echo " Cap is host-side: softirq/CPU/bridge/NIC coalescing."
|
||||
fi
|
||||
echo "==================================================================="
|
||||
else
|
||||
analyze "Sender TX (control: should split if host hash=layer3+4)" 3 "$TMP/before" "$TMP/after"
|
||||
dom="$(dominant 3 "$TMP/before" "$TMP/after")"
|
||||
pct=${dom%% *}; top=${dom##* }
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
|
||||
echo " VERDICT: host TX NOT distributing (top slave '$top' = ${pct}%)."
|
||||
echo " Host xmit_hash_policy is NOT effective despite /proc/net/bonding."
|
||||
else
|
||||
echo " VERDICT: host TX distributing OK (top slave '$top' = ${pct}%)."
|
||||
fi
|
||||
echo "==================================================================="
|
||||
fi
|
||||
} | tee -a "$LOG"
|
||||
|
||||
echo ""
|
||||
echo "Log written: $LOG"
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# run-lacp-retrans-cause.sh (workstation wrapper)
|
||||
#
|
||||
# Deploys scripts/lacp-retrans-cause.sh to BOTH hosts, runs them in the
|
||||
# right order, scps logs back, prints them.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
RECV="pfv-tsys6"; SEND="pfv-tsys7"
|
||||
LOCAL_LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
||||
SCRIPT="lacp-retrans-cause.sh"
|
||||
LOCAL_SCRIPT="/home/reachableceo/projects/perfopt/scripts/${SCRIPT}"
|
||||
REMOTE_SCRIPT="/root/${SCRIPT}"
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " LACP retransmit-cause investigation"
|
||||
echo " Receiver: $RECV Sender: $SEND"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
echo "--- preflight ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
ssh "${SSH[@]}" "root@$h" \
|
||||
'command -v ethtool >/dev/null && e=OK || e=MISSING
|
||||
command -v iperf3 >/dev/null && i=OK || i=MISSING
|
||||
printf "ethtool=%s iperf3=%s host=%s\n" "$e" "$i" "$(uname -n)"' 2>&1 | head -1
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- deploy ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
||||
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
||||
&& echo "deployed" || echo "FAILED"
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- cleanup stale iperf3 ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- launching receiver on $RECV (background) ---"
|
||||
ssh "${SSH[@]}" "root@$RECV" \
|
||||
"nohup bash $REMOTE_SCRIPT > /root/lacp-retrans-receiver.console 2>&1 &" 2>/dev/null
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
||||
ssh "${SSH[@]}" "root@$SEND" "bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
||||
echo ""
|
||||
sleep 3
|
||||
|
||||
echo "--- fetching logs ---"
|
||||
for f in lacp-retrans-receiver.log lacp-retrans-receiver.console lacp-retrans-sender.log; do
|
||||
src=""
|
||||
case "$f" in
|
||||
*receiver*) src="$RECV" ;;
|
||||
*sender*) src="$SEND" ;;
|
||||
esac
|
||||
printf ' %-32s <- %s : ' "$f" "$src"
|
||||
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
||||
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
||||
else
|
||||
echo "MISSING"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "==================================================================="
|
||||
echo " RECEIVER LOG ($RECV)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-retrans-receiver.log" 2>/dev/null || echo "(missing)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SENDER LOG ($SEND)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-retrans-sender.log" 2>/dev/null || echo "(missing)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Local copies in: $LOCAL_LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# run-lacp-rx-distribution.sh (workstation wrapper)
|
||||
#
|
||||
# One-shot orchestrator: deploys scripts/lacp-rx-distribution.sh to BOTH
|
||||
# tsys6 (receiver) and tsys7 (sender), runs them in the right order, then
|
||||
# scps both logs back to returned-logs/iperf/.
|
||||
#
|
||||
# Run from the workstation:
|
||||
# bash scripts/run-lacp-rx-distribution.sh
|
||||
#
|
||||
# Idempotent: safe to re-run. iperf3 is killed on both hosts first.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
RECV="pfv-tsys6" # receiver (we care most about its RX split)
|
||||
SEND="pfv-tsys7" # sender (control: its TX split)
|
||||
LOCAL_LOG_DIR="/home/reachableceo/projects/perfopt/returned-logs/iperf"
|
||||
SCRIPT="lacp-rx-distribution.sh"
|
||||
LOCAL_SCRIPT="/home/reachableceo/projects/perfopt/scripts/${SCRIPT}"
|
||||
REMOTE_SCRIPT="/root/${SCRIPT}"
|
||||
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " LACP per-slave RX/TX distribution test"
|
||||
echo " Receiver: $RECV (10.100.100.6) Sender: $SEND (10.100.100.7)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# 0. Preflight: confirm SSH and iperf3 on both hosts
|
||||
echo "--- preflight (ssh + iperf3 + bond0) ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
ssh "${SSH[@]}" "root@$h" \
|
||||
'command -v iperf3 >/dev/null && ip=$(command -v iperf3) || ip=MISSING
|
||||
[ -r /proc/net/bonding/bond0 ] && b=OK || b=NO-BOND0
|
||||
printf "iperf3=%s bond0=%s host=%s\n" "$ip" "$b" "$(uname -n)"' \
|
||||
2>&1 | head -1
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 1. Copy the script to both hosts + chmod
|
||||
echo "--- deploy $SCRIPT to both hosts ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
||||
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
||||
&& echo "deployed + chmod +x" \
|
||||
|| echo "DEPLOY FAILED"
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 2. Kill any stale iperf3 on both hosts
|
||||
echo "--- cleanup stale iperf3 ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 3. Start RECEIVER in background (one-shot server, writes /root/lacp-rx-receiver.log)
|
||||
echo "--- starting receiver on $RECV (background) ---"
|
||||
ssh "${SSH[@]}" "root@$RECV" \
|
||||
"nohup bash $REMOTE_SCRIPT > /root/lacp-rx-receiver.console 2>&1 &" 2>/dev/null
|
||||
echo " receiver launched; waiting 5s for it to start iperf3 -s -1 ..."
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
# 4. Run SENDER (foreground; ~15s with the default 12s test + 3s pre-sleep)
|
||||
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
||||
ssh "${SSH[@]}" "root@$SEND" \
|
||||
"bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
||||
echo ""
|
||||
|
||||
# 5. Give receiver a moment to finish writing its log
|
||||
sleep 3
|
||||
|
||||
# 6. Fetch logs back
|
||||
echo "--- fetching logs ---"
|
||||
for f in lacp-rx-receiver.log lacp-rx-receiver.console lacp-rx-sender.log; do
|
||||
src=""
|
||||
case "$f" in
|
||||
lacp-rx-receiver*) src="$RECV" ;;
|
||||
lacp-rx-sender*) src="$SEND" ;;
|
||||
esac
|
||||
printf ' %-28s <- %s : ' "$f" "$src"
|
||||
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
||||
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
||||
else
|
||||
echo "MISSING"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 7. Show the receiver log (the decisive one)
|
||||
echo "==================================================================="
|
||||
echo " RECEIVER LOG ($RECV — the decisive side)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-rx-receiver.log" 2>/dev/null || echo "(no log)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SENDER LOG ($SEND — control)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-rx-sender.log" 2>/dev/null || echo "(no log)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Local copies in: $LOCAL_LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
Executable
+127
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
# sw-capture-remote.sh - orchestrate a serial capture from this workstation.
|
||||
#
|
||||
# Flow:
|
||||
# 1. De-conflict: abort if any local ssh to pfv-tsys4 is in flight
|
||||
# (other agent could be there).
|
||||
# 2. Free the serial port: kill whatever holds /dev/ttyUSBx
|
||||
# (typically a screen session). Targeted, not blanket.
|
||||
# 3. scp driver + .cmds to pfv-tsys4.
|
||||
# 4. Run driver over ssh, capture stderr to console.
|
||||
# 5. scp the resulting log back to returned-logs/.
|
||||
#
|
||||
# Usage:
|
||||
# sw-capture-remote.sh <switch-name> [device]
|
||||
#
|
||||
# <switch-name> e.g. pfv-core-sw01 (must have switches/<name>.cmds)
|
||||
# [device] /dev/ttyUSBx on pfv-tsys4. Defaults per switch map below.
|
||||
#
|
||||
# Currently scoped to pfv-core-sw01 only (per user direction). The other
|
||||
# two switches are deferred; their defaults are placeholders.
|
||||
set -u
|
||||
|
||||
SWITCH=${1:-}
|
||||
DEVICE=${2:-}
|
||||
|
||||
if [ -z "$SWITCH" ]; then
|
||||
echo "Usage: $0 <switch-name> [device]" >&2
|
||||
echo " e.g. $0 pfv-core-sw01 /dev/ttyUSB2" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Switch -> default device map (ttyUSB2 = core-sw01 confirmed by user).
|
||||
case "$SWITCH" in
|
||||
pfv-core-sw01)
|
||||
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB2 ;;
|
||||
pfv-r3-tor-mgmt)
|
||||
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB0 # TENTATIVE - unconfirmed
|
||||
if [ "${2:-}" = "" ]; then
|
||||
echo "NOTE: pfv-r3-tor-mgmt device is tentative (/dev/ttyUSB0)." >&2
|
||||
echo " Pass the device explicitly if different." >&2
|
||||
fi ;;
|
||||
pfv-r3-tor-stor)
|
||||
[ -z "$DEVICE" ] && DEVICE=/dev/ttyUSB1 # TENTATIVE - unconfirmed
|
||||
if [ "${2:-}" = "" ]; then
|
||||
echo "NOTE: pfv-r3-tor-stor device is tentative (/dev/ttyUSB1)." >&2
|
||||
echo " Pass the device explicitly if different." >&2
|
||||
fi ;;
|
||||
*)
|
||||
echo "unknown switch: $SWITCH" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
BAUD=9600
|
||||
HOST=root@pfv-tsys4
|
||||
HERE=/home/reachableceo/projects/perfopt
|
||||
LOCAL_DRIVER=$HERE/scripts/sw-capture.py
|
||||
LOCAL_CMDS=$HERE/switches/$SWITCH.cmds
|
||||
LOCAL_LOG=$HERE/returned-logs/$SWITCH.log
|
||||
REMOTE_DRIVER=/root/sw-capture.py
|
||||
REMOTE_CMDS=/root/$SWITCH.cmds
|
||||
REMOTE_LOG=/root/$SWITCH.log
|
||||
|
||||
[ -f "$LOCAL_DRIVER" ] || { echo "missing $LOCAL_DRIVER" >&2; exit 2; }
|
||||
[ -f "$LOCAL_CMDS" ] || { echo "missing $LOCAL_CMDS" >&2; exit 2; }
|
||||
|
||||
ts() { date +%H:%M:%S; }
|
||||
|
||||
echo "[$(ts)] switch=$SWITCH device=$DEVICE baud=$BAUD host=$HOST"
|
||||
|
||||
# 1. De-conflict: any local ssh to pfv-tsys4 in flight?
|
||||
echo "[$(ts)] checking for in-flight ssh to pfv-tsys4..."
|
||||
if ps -eo pid,etime,args | grep -E 'ssh.*pfv-tsys4|scp.*pfv-tsys4' | grep -v grep >/tmp/.swcap.ps 2>&1; then
|
||||
cat /tmp/.swcap.ps
|
||||
echo "[$(ts)] ABORT: another ssh/scp to pfv-tsys4 is running (other agent?)." >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[$(ts)] clear."
|
||||
rm -f /tmp/.swcap.ps
|
||||
|
||||
# 2. Free the serial port: kill whatever holds $DEVICE.
|
||||
echo "[$(ts)] freeing $DEVICE on $HOST (targeted; other screen sessions untouched)..."
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=8 "$HOST" \
|
||||
"fuser -v $DEVICE 2>&1 | tee /dev/stderr; \
|
||||
fuser -k -TERM $DEVICE 2>/dev/null; sleep 1; \
|
||||
if fuser $DEVICE 2>/dev/null; then \
|
||||
echo 'still held after SIGTERM, escalating to SIGKILL'; \
|
||||
fuser -k -KILL $DEVICE 2>/dev/null; sleep 1; \
|
||||
fi; \
|
||||
fuser $DEVICE 2>/dev/null && echo 'STILL HELD' || echo 'FREE'"
|
||||
|
||||
# Re-check; abort if still held.
|
||||
HELD=$(ssh -o BatchMode=yes "$HOST" "fuser $DEVICE 2>/dev/null && echo HELD || echo FREE")
|
||||
if [ "$HELD" = "HELD" ]; then
|
||||
echo "[$(ts)] ABORT: $DEVICE still held on $HOST." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 3. Copy driver + cmds.
|
||||
echo "[$(ts)] copying driver + cmds to $HOST..."
|
||||
scp -q "$LOCAL_DRIVER" "$HOST:$REMOTE_DRIVER"
|
||||
scp -q "$LOCAL_CMDS" "$HOST:$REMOTE_CMDS"
|
||||
|
||||
# 4. Run the capture on pfv-tsys4. Stream stderr (progress) to console.
|
||||
echo "[$(ts)] running capture..."
|
||||
ssh -o BatchMode=yes -o ServerAliveInterval=10 "$HOST" \
|
||||
"python3 $REMOTE_DRIVER \
|
||||
--device $DEVICE --baud $BAUD \
|
||||
--cmds $REMOTE_CMDS --log $REMOTE_LOG"
|
||||
RC=$?
|
||||
echo "[$(ts)] capture exit code: $RC"
|
||||
|
||||
# 5. Pull log back.
|
||||
echo "[$(ts)] pulling log back to $LOCAL_LOG..."
|
||||
mkdir -p "$(dirname "$LOCAL_LOG")"
|
||||
scp -q "$HOST:$REMOTE_LOG" "$LOCAL_LOG"
|
||||
if [ -f "$LOCAL_LOG" ]; then
|
||||
SZ=$(wc -c < "$LOCAL_LOG")
|
||||
echo "[$(ts)] OK: $LOCAL_LOG ($SZ bytes)"
|
||||
echo "----- head -----"
|
||||
head -30 "$LOCAL_LOG"
|
||||
echo "----- tail -----"
|
||||
tail -10 "$LOCAL_LOG"
|
||||
else
|
||||
echo "[$(ts)] ERROR: log not pulled back." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
exit $RC
|
||||
@@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
sw-capture.py - drive a Dell switch over a serial console and log all output.
|
||||
|
||||
Read-only. Sends ONLY the commands listed in the supplied .cmds file
|
||||
(comment lines starting with '!' and blank lines are skipped). Handles
|
||||
`--More--` pagination by sending a space. Aborts cleanly on any password
|
||||
prompt (we never supply credentials).
|
||||
|
||||
Pure stdlib (termios + select). No pyserial/expect required.
|
||||
|
||||
Exit codes:
|
||||
0 clean run, every command saw a prompt again
|
||||
2 could not synchronize with a prompt during wake
|
||||
3 one or more commands timed out (log still written)
|
||||
4 password prompt encountered (aborted)
|
||||
|
||||
Usage:
|
||||
sw-capture.py --device /dev/ttyUSB2 --baud 9600 \\
|
||||
--cmds pfv-core-sw01.cmds --log /root/pfv-core-sw01.log
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import select
|
||||
import sys
|
||||
import termios
|
||||
import time
|
||||
|
||||
PROMPT_RE = re.compile(rb'[>#]\s*$') # ends in # or > + spaces
|
||||
MORE_RE = re.compile(rb'--\s*More\s*--') # pagination prompt
|
||||
PWD_RE = re.compile(rb'[Pp]assword:\s*$') # enable / login password
|
||||
|
||||
BAUDS = {
|
||||
'9600': termios.B9600,
|
||||
'19200': termios.B19200,
|
||||
'38400': termios.B38400,
|
||||
'57600': termios.B57600,
|
||||
'115200': termios.B115200,
|
||||
}
|
||||
|
||||
|
||||
def log(msg, level='INFO'):
|
||||
sys.stderr.write(f'[{level}] {msg}\n')
|
||||
sys.stderr.flush()
|
||||
|
||||
|
||||
def open_port(device, baud):
|
||||
"""Open the serial device raw at the requested baud, 8N1, no flow ctrl."""
|
||||
fd = os.open(device, os.O_RDWR | os.O_NOCTTY)
|
||||
try:
|
||||
attrs = termios.tcgetattr(fd)
|
||||
except termios.error:
|
||||
log(f'{device} is not a termios-capable device', 'WARN')
|
||||
return fd
|
||||
|
||||
# raw input
|
||||
attrs[0] &= ~(termios.IGNBRK | termios.BRKINT | termios.PARMRK |
|
||||
termios.ISTRIP | termios.INLCR | termios.IGNCR |
|
||||
termios.ICRNL | termios.IXON)
|
||||
# raw output
|
||||
attrs[1] &= ~termios.OPOST
|
||||
# 8N1, enable receiver, ignore modem control lines
|
||||
attrs[2] &= ~(termios.CSIZE | termios.PARENB | termios.CSTOPB)
|
||||
attrs[2] |= termios.CS8 | termios.CREAD | termios.CLOCAL
|
||||
# raw local
|
||||
attrs[3] &= ~(termios.ECHO | termios.ECHONL | termios.ICANON |
|
||||
termios.ISIG | termios.IEXTEN)
|
||||
# non-blocking-ish reads (select is the primary gate)
|
||||
attrs[6][termios.VMIN] = 0
|
||||
attrs[6][termios.VTIME] = 1
|
||||
|
||||
b = BAUDS.get(str(baud))
|
||||
if b is None:
|
||||
raise SystemExit(f'unsupported baud: {baud}')
|
||||
# Set ispeed/ospeed directly on the attribute list. (Equivalent to
|
||||
# termios.cfsetispeed/cfsetospeed, which are missing on some Python
|
||||
# builds — e.g. the one on pfv-tsys4.)
|
||||
attrs[4] = b # ispeed
|
||||
attrs[5] = b # ospeed
|
||||
termios.tcsetattr(fd, termios.TCSANOW, attrs)
|
||||
return fd
|
||||
|
||||
|
||||
def read_chunk(fd, timeout):
|
||||
"""Read whatever arrives within `timeout`. Extends briefly on activity."""
|
||||
buf = b''
|
||||
deadline = time.time() + timeout
|
||||
while True:
|
||||
remaining = deadline - time.time()
|
||||
if remaining <= 0:
|
||||
return buf
|
||||
r, _, _ = select.select([fd], [], [], min(0.5, remaining))
|
||||
if not r:
|
||||
if buf:
|
||||
return buf
|
||||
continue
|
||||
try:
|
||||
chunk = os.read(fd, 4096)
|
||||
except OSError:
|
||||
return buf
|
||||
if not chunk:
|
||||
return buf
|
||||
buf += chunk
|
||||
# keep collecting as long as bytes are flowing
|
||||
deadline = time.time() + 0.3
|
||||
|
||||
|
||||
def drain(fd, timeout=1.0):
|
||||
total = 0
|
||||
while True:
|
||||
b = read_chunk(fd, timeout=timeout)
|
||||
if not b:
|
||||
return total
|
||||
total += len(b)
|
||||
|
||||
|
||||
def send(fd, s):
|
||||
if isinstance(s, str):
|
||||
s = s.encode()
|
||||
os.write(fd, s)
|
||||
|
||||
|
||||
def wait_for(fd, regex, timeout, on_more=None, on_pwd=None):
|
||||
"""Read until `regex` matches the tail of the buffer, or timeout."""
|
||||
buf = b''
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
remaining = deadline - time.time()
|
||||
chunk = read_chunk(fd, timeout=min(1.0, remaining))
|
||||
if chunk:
|
||||
buf += chunk
|
||||
tail64 = buf[-64:]
|
||||
tail32 = buf[-32:]
|
||||
tail128 = buf[-128:]
|
||||
if on_more and MORE_RE.search(tail64):
|
||||
on_more(fd)
|
||||
continue
|
||||
if on_pwd and PWD_RE.search(tail32):
|
||||
on_pwd(buf)
|
||||
return buf, 'pwd'
|
||||
if regex.search(tail128):
|
||||
return buf, 'ok'
|
||||
return buf, 'timeout'
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument('--device', required=True)
|
||||
ap.add_argument('--baud', type=int, default=9600)
|
||||
ap.add_argument('--cmds', required=True)
|
||||
ap.add_argument('--log', required=True)
|
||||
ap.add_argument('--per-cmd-timeout', type=float, default=45.0)
|
||||
ap.add_argument('--wake-timeout', type=float, default=15.0)
|
||||
ap.add_argument('--session-max', type=float, default=600.0)
|
||||
args = ap.parse_args()
|
||||
|
||||
cmds = []
|
||||
with open(args.cmds) as f:
|
||||
for raw in f:
|
||||
s = raw.rstrip('\n').strip()
|
||||
if not s or s.startswith('!'):
|
||||
continue
|
||||
cmds.append(s)
|
||||
log(f'parsed {len(cmds)} commands from {args.cmds}')
|
||||
|
||||
logf = open(args.log, 'wb', buffering=0)
|
||||
|
||||
def w(b):
|
||||
if isinstance(b, str):
|
||||
b = b.encode()
|
||||
logf.write(b)
|
||||
|
||||
w(f'==== sw-capture {time.strftime("%Y-%m-%d %H:%M:%S")} ====\n')
|
||||
w(f'device={args.device} baud={args.baud} cmds={args.cmds} '
|
||||
f'n={len(cmds)} per_cmd_timeout={args.per_cmd_timeout}\n\n')
|
||||
|
||||
fd = open_port(args.device, args.baud)
|
||||
log(f'opened {args.device} @ {args.baud} 8N1 raw')
|
||||
|
||||
session_start = time.time()
|
||||
abort = False
|
||||
|
||||
def on_more(fd_):
|
||||
log('--More-- -> space')
|
||||
send(fd_, b' ')
|
||||
|
||||
def on_pwd(buf):
|
||||
nonlocal abort
|
||||
abort = True
|
||||
log('password prompt detected (enable or login) - aborting; '
|
||||
'no credentials supplied', 'ERROR')
|
||||
w(buf)
|
||||
w(b'\n[PASSWORD PROMPT - ABORTED]\n')
|
||||
|
||||
# WAKE: nudge with Ctrl-C + Enter, look for any prompt
|
||||
drain(fd, 0.5)
|
||||
synced = False
|
||||
wake_deadline = time.time() + args.wake_timeout
|
||||
attempt = 0
|
||||
while time.time() < wake_deadline:
|
||||
attempt += 1
|
||||
send(fd, b'\x03')
|
||||
time.sleep(0.2)
|
||||
send(fd, b'\r')
|
||||
buf, status = wait_for(fd, PROMPT_RE, timeout=3.0,
|
||||
on_more=on_more, on_pwd=on_pwd)
|
||||
w(buf)
|
||||
if status == 'pwd':
|
||||
logf.close(); os.close(fd); sys.exit(4)
|
||||
if status == 'ok':
|
||||
synced = True
|
||||
log(f'prompt synced after {attempt} attempt(s)')
|
||||
break
|
||||
if not synced:
|
||||
w(b'\n[NO PROMPT - ABORT]\n')
|
||||
log('no prompt detected during wake window', 'ERROR')
|
||||
logf.close(); os.close(fd); sys.exit(2)
|
||||
|
||||
# RUN commands verbatim from the .cmds list
|
||||
failures = 0
|
||||
for idx, cmd in enumerate(cmds, 1):
|
||||
if time.time() - session_start > args.session_max:
|
||||
log('session_max exceeded - stopping early', 'ERROR')
|
||||
w(b'\n[SESSION_MAX - STOP]\n')
|
||||
break
|
||||
if abort:
|
||||
break
|
||||
log(f'[{idx}/{len(cmds)}] {cmd}')
|
||||
send(fd, cmd + '\r')
|
||||
buf, status = wait_for(fd, PROMPT_RE,
|
||||
timeout=args.per_cmd_timeout,
|
||||
on_more=on_more, on_pwd=on_pwd)
|
||||
w(buf)
|
||||
if status == 'pwd':
|
||||
failures += 1
|
||||
break
|
||||
if status == 'timeout':
|
||||
log(f'timeout after: {cmd}', 'WARN')
|
||||
failures += 1
|
||||
# try to resync: Ctrl-C + drain
|
||||
send(fd, b'\x03')
|
||||
time.sleep(0.3)
|
||||
drain(fd, 0.5)
|
||||
|
||||
w(f'\n==== end {time.strftime("%Y-%m-%d %H:%M:%S")} '
|
||||
f'failures={failures} ====\n')
|
||||
logf.close()
|
||||
os.close(fd)
|
||||
log(f'done -> {args.log} failures={failures}')
|
||||
sys.exit(0 if failures == 0 else 3)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
#!/usr/bin/env bash
|
||||
# Probe conman state + expect availability on pfv-tsys4.
|
||||
# Read-only. Decides whether we drive via conman+expect or expect-only.
|
||||
set -u
|
||||
|
||||
# De-conflict: any ssh to pfv-tsys4 right now?
|
||||
echo "===== LOCAL ssh activity ====="
|
||||
ps -eo pid,etime,args | grep -E 'ssh.*pfv-tsys' | grep -v grep || echo "(none to pfv-tsys4)"
|
||||
|
||||
echo
|
||||
echo "===== pfv-tsys4: conman + expect state ====="
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=5 root@pfv-tsys4 'bash -s' <<'REMOTE'
|
||||
echo "--- conmand service ---"
|
||||
systemctl is-active conmand 2>&1 || true
|
||||
systemctl is-enabled conmand 2>&1 || true
|
||||
systemctl status conmand --no-pager 2>&1 | head -15 || true
|
||||
|
||||
echo
|
||||
echo "--- conman binary ---"
|
||||
command -v conman && conman --version 2>&1 | head -2 || echo "conman: MISSING"
|
||||
command -v conmand && echo "conmand present" || echo "conmand: MISSING"
|
||||
|
||||
echo
|
||||
echo "--- /etc/conman.conf: ttyUSB2 entries ---"
|
||||
grep -nE "ttyUSB2|core-sw|CONSOLE|LOG|SERIAL|BAUD" /etc/conman.conf 2>/dev/null | head -40 || echo "(no matches / no file)"
|
||||
|
||||
echo
|
||||
echo "--- conman log dir ---"
|
||||
ls -la /var/log/conman/ 2>&1 | head -20 || echo "(no /var/log/conman)"
|
||||
ls -la /var/consoles/ 2>&1 | head -20 || echo "(no /var/consoles)"
|
||||
|
||||
echo
|
||||
echo "--- expect availability ---"
|
||||
command -v expect && expect -v 2>&1 || echo "expect: NOT installed"
|
||||
echo "apt-cache policy expect:"
|
||||
apt-cache policy expect 2>/dev/null | head -10 || echo "(apt-cache failed)"
|
||||
|
||||
echo
|
||||
echo "--- other useful drivers ---"
|
||||
for t in tclsh socat cu tip; do
|
||||
command -v "$t" 2>/dev/null && echo " $t: present" || true
|
||||
done
|
||||
|
||||
echo
|
||||
echo "--- apt network reachability (quick) ---"
|
||||
timeout 5 bash -c 'echo > /dev/tcp/deb.debian.org/80' 2>&1 && echo "apt network: OK" || echo "apt network: UNREACHABLE"
|
||||
|
||||
echo
|
||||
echo "--- disk space for log ---"
|
||||
df -h /root 2>&1 | tail -2
|
||||
|
||||
echo
|
||||
echo "--- screen sessions (still 3?) ---"
|
||||
screen -ls 2>&1 || true
|
||||
REMOTE
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
# Probe pfv-tsys4 for what's available to drive the serial console.
|
||||
# Also snapshots local ssh/scp activity so we can de-conflict with the
|
||||
# other agent running in this directory.
|
||||
set -u
|
||||
|
||||
echo "===== LOCAL ssh/scp activity (other-agent de-confliction) ====="
|
||||
ps -eo pid,ppid,etime,user,args | grep -E 'ssh|scp' | grep -v grep || echo "(none)"
|
||||
|
||||
echo
|
||||
echo "===== Ping pfv-tsys4 ====="
|
||||
ping -c1 -W2 pfv-tsys4 >/dev/null 2>&1 && echo "ping OK" || echo "ping FAIL"
|
||||
|
||||
echo
|
||||
echo "===== Probe pfv-tsys4 over ssh ====="
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=5 root@pfv-tsys4 'bash -s' <<'REMOTE'
|
||||
echo "--- host ---"
|
||||
hostname; uname -a
|
||||
echo "--- tools ---"
|
||||
for t in python3 python expect screen minicom picocom stty fuser lsof; do
|
||||
p=$(command -v "$t" 2>/dev/null) && echo "$t -> $p" || echo "$t -> MISSING"
|
||||
done
|
||||
echo "--- pyserial ---"
|
||||
python3 -c "import serial; print('pyserial', serial.__version__)" 2>&1
|
||||
echo "--- device node ---"
|
||||
ls -l /dev/ttyUSB2 2>&1
|
||||
stat -c '%n owner=%U:%G mode=%a' /dev/ttyUSB2 2>&1 || true
|
||||
echo "--- who holds /dev/ttyUSB2 ---"
|
||||
fuser -v /dev/ttyUSB2 2>&1 || echo "(fuser: none or n/a)"
|
||||
lsof /dev/ttyUSB2 2>&1 | head -20 || true
|
||||
echo "--- screen sessions on this host ---"
|
||||
screen -ls 2>&1 || echo "(no screen / not installed)"
|
||||
echo "--- current tty settings (only readable if not held exclusively) ---"
|
||||
stty -F /dev/ttyUSB2 2>&1 || echo "(held exclusively - expected if screen is up)"
|
||||
echo "--- baud hints in config/history ---"
|
||||
grep -riE "ttyUSB2|115200|9600|baud" /etc/ ~/.screenrc ~/.bash_history 2>/dev/null | head -20 || true
|
||||
echo "--- recent console-related processes ---"
|
||||
ps -eo pid,etime,user,args | grep -E 'screen|minicom|picocom|ttyUSB' | grep -v grep || echo "(none)"
|
||||
REMOTE
|
||||
Reference in New Issue
Block a user