prep for next ai session
This commit is contained in:
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
# check-pkgs.sh - verify package install state.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
for host in pfv-tsys6 pfv-tsys7; do
|
||||
echo "=== $host ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" '
|
||||
for p in sysstat jq numactl nvme mtr-tiny dig bmon tcpdump; do
|
||||
if command -v "$p" >/dev/null 2>&1; then
|
||||
echo " ✓ $p"
|
||||
else
|
||||
echo " ✗ $p"
|
||||
fi
|
||||
done
|
||||
# sysstat config
|
||||
echo " sysstat service:"
|
||||
systemctl list-unit-files 2>/dev/null | grep -i sysstat | sed "s/^/ /"
|
||||
echo " sysstat enabled in /etc/default:"
|
||||
if [ -r /etc/default/sysstat ]; then
|
||||
grep ENABLED /etc/default/sysstat | sed "s/^/ /"
|
||||
else
|
||||
echo " no /etc/default/sysstat"
|
||||
fi
|
||||
# on Debian trixie, sysstat uses a different path
|
||||
ls /etc/cron.d/sysstat* 2>/dev/null | sed "s/^/ found: /"
|
||||
'
|
||||
echo ""
|
||||
done
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
# check-repos-and-reboot.sh - checks reboot-required + Proxmox repo config on all hosts.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7)
|
||||
|
||||
for host in "${HOSTS[@]}"; do
|
||||
echo "================================================================"
|
||||
echo "[$host]"
|
||||
echo "================================================================"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "--- /var/run/reboot-required ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'if [ -r /var/run/reboot-required ]; then echo "REBOOT REQUIRED"; cat /var/run/reboot-required 2>/dev/null; if [ -r /var/run/reboot-required.pkgs ]; then echo "Packages triggering:"; cat /var/run/reboot-required.pkgs; fi; else echo "(no reboot required marker)"; fi'
|
||||
|
||||
echo ""
|
||||
echo "--- Running kernel vs installed kernel ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'echo "running: $(uname -r)"; echo "installed:"; dpkg -l | grep -E "pve-kernel-[0-9]" | awk "{print \" \"\$2\" \"\$3}" | tail -5'
|
||||
|
||||
echo ""
|
||||
echo "--- Proxmox repositories (apt sources) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -rh "pve\|proxmox" /etc/apt/sources.list /etc/apt/sources.list.d/ 2>/dev/null | grep -v "^#" | sed "s/^/ /"'
|
||||
|
||||
echo ""
|
||||
echo "--- Enterprise repo status (should be commented or absent if no subscription) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -l "pve-enterprise" /etc/apt/sources.list /etc/apt/sources.list.d/* 2>/dev/null | while read f; do echo " File: $f"; grep -n "pve-enterprise" "$f" | sed "s/^/ /"; done'
|
||||
|
||||
echo ""
|
||||
echo "--- no-subscription repo presence ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -rl "pve-no-subscription" /etc/apt/sources.list /etc/apt/sources.list.d/ 2>/dev/null | while read f; do echo " File: $f"; grep -n "pve-no-subscription" "$f" | sed "s/^/ /"; done'
|
||||
|
||||
echo ""
|
||||
echo "--- Recently updated packages (last 24h, kernel-related) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -E "pve-kernel|proxmox|pve-qemu|zfs" /var/log/dpkg.log 2>/dev/null | grep "$(date +%Y-%m-%d)\|$(date -d yesterday +%Y-%m-%d)" | tail -15 || echo "(none in dpkg.log)"'
|
||||
|
||||
echo ""
|
||||
done
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/bin/bash
|
||||
# deploy-and-fix.sh - uploads fix script, runs it, starts VMs, verifies.
|
||||
set -uo pipefail
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scripts"
|
||||
|
||||
echo "=== Uploading fix script to $HOST ==="
|
||||
scp "${SSH_OPTS[@]}" "$SCRIPT_DIR/fix-bond-nfs.sh" "root@$HOST:/root/fix-bond-nfs.sh" >/dev/null 2>&1
|
||||
|
||||
echo "=== Running fix ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'chmod +x /root/fix-bond-nfs.sh && bash /root/fix-bond-nfs.sh' 2>&1
|
||||
|
||||
echo ""
|
||||
echo "=== Starting VMs ==="
|
||||
for vmid in $(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1{print \$1}"'); do
|
||||
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
||||
if [ "$status" != "running" ]; then
|
||||
echo " Starting VM $vmid..."
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1 | sed 's/^/ /'
|
||||
else
|
||||
echo " VM $vmid already running"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Waiting 20s for VMs to boot..."
|
||||
sleep 20
|
||||
|
||||
echo ""
|
||||
echo "=== FULL VERIFICATION ==="
|
||||
echo ""
|
||||
echo "--- VMs ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
||||
echo ""
|
||||
echo "--- NFS mounts ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -24'
|
||||
echo ""
|
||||
echo "--- NFS TCP connections ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||||
echo " Count:"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l'
|
||||
echo ""
|
||||
echo "--- bond0 hash policy ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/net/bonding/bond0 | head -6'
|
||||
echo ""
|
||||
echo "--- Summary ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo "tcp_cc: $(sysctl -n net.ipv4.tcp_congestion_control)"'
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo "swappiness: $(sysctl -n vm.swappiness)"'
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo "governor: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null)"'
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'tuned-adm active 2>/dev/null'
|
||||
Executable
+174
@@ -0,0 +1,174 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# deploy-check.sh
|
||||
#
|
||||
# Deploys scripts/check.sh to each reachable Proxmox host, executes it
|
||||
# read-only, and pulls the resulting log back to returned-logs/.
|
||||
#
|
||||
# EXPLICITLY SKIPS:
|
||||
# - pfv-tsys2 (off the air per user; Win10 pending rebuild)
|
||||
# - pfv-tsys8 (retired / permanently offline per user)
|
||||
#
|
||||
# Safety features:
|
||||
# - BatchMode=yes : never hang on a password prompt
|
||||
# - ConnectTimeout=8 : fail fast on dead hosts
|
||||
# - per-host try/skip : one bad host never aborts the run
|
||||
# - ServerAliveInterval : detect hung connections
|
||||
# - read-only script : check.sh modifies nothing on the target
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd "$SELF_DIR/.." && pwd)"
|
||||
CHECK_SH="$SELF_DIR/scripts/check.sh"
|
||||
LOG_DIR="$ROOT_DIR/returned-logs"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
# ONLY the hosts the user told us are alive.
|
||||
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7 pfv-tsys9)
|
||||
|
||||
# Common ssh options: non-interactive, fail-fast, no host-key prompt blocking.
|
||||
SSH_OPTS=(-o BatchMode=yes
|
||||
-o ConnectTimeout=8
|
||||
-o ServerAliveInterval=10
|
||||
-o ServerAliveCountMax=3
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
|
||||
log() { printf '[%s] %s\n' "$(date +%H:%M:%S)" "$*"; }
|
||||
|
||||
if [ ! -r "$CHECK_SH" ]; then
|
||||
echo "FATAL: $CHECK_SH not found" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Sanity-check shellcheck clean before shipping (best-effort, non-blocking)
|
||||
if command -v docker >/dev/null 2>&1; then
|
||||
log "pre-flight: shellcheck on check.sh"
|
||||
if ! docker run --rm -v "$SELF_DIR:/mnt" -w /mnt \
|
||||
koalaman/shellcheck:stable --severity=style --format=gcc scripts/check.sh \
|
||||
>"$LOG_DIR/_shellcheck.preflight.txt" 2>&1; then
|
||||
log "WARNING: shellcheck reported issues — see _shellcheck.preflight.txt"
|
||||
log " aborting deploy to avoid shipping a broken script"
|
||||
exit 1
|
||||
fi
|
||||
log "pre-flight: shellcheck clean"
|
||||
fi
|
||||
|
||||
summary_pass=()
|
||||
summary_fail=()
|
||||
declare -A HOST_PID # host -> background pid
|
||||
declare -A HOST_MARKER # host -> per-host marker file
|
||||
|
||||
# Per-host worker — runs in background, one per host, all in parallel.
|
||||
# Writes status into a marker file consumed by the parent.
|
||||
worker() {
|
||||
local host="$1"
|
||||
local marker="$LOG_DIR/_marker.$host"
|
||||
: > "$marker" # truncate
|
||||
echo "running" >> "$marker"
|
||||
|
||||
local short=""
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "fail unreachable" >> "$marker"
|
||||
return
|
||||
fi
|
||||
if ! scp "${SSH_OPTS[@]}" "$CHECK_SH" "root@$host:/root/check.sh" >/dev/null 2>&1; then
|
||||
echo "fail scp-upload-failed" >> "$marker"
|
||||
return
|
||||
fi
|
||||
local remote_size
|
||||
remote_size=$(ssh "${SSH_OPTS[@]}" "root@$host" 'wc -c < /root/check.sh' 2>/dev/null || echo 0)
|
||||
if [ "${remote_size:-0}" -lt 1000 ]; then
|
||||
echo "fail upload-corrupt" >> "$marker"
|
||||
return
|
||||
fi
|
||||
local remote_stdout
|
||||
remote_stdout=$(ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
'chmod +x /root/check.sh && bash /root/check.sh' 2>&1)
|
||||
local rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "fail check-exit-$rc" >> "$marker"
|
||||
# don't return - still try to pull whatever log got produced
|
||||
fi
|
||||
short=$(printf '%s\n' "$remote_stdout" | grep -oE 'Wrote: /root/[a-zA-Z0-9_-]+\.log' | head -n1 | awk '{print $2}')
|
||||
if [ -z "$short" ]; then
|
||||
short=$(ssh "${SSH_OPTS[@]}" "root@$host" 'echo "/root/$(hostname -s).log"' 2>/dev/null)
|
||||
fi
|
||||
if [ -z "$short" ]; then
|
||||
echo "fail no-log-path" >> "$marker"
|
||||
return
|
||||
fi
|
||||
if ! scp "${SSH_OPTS[@]}" "root@$host:$short" "$LOG_DIR/" >/dev/null 2>&1; then
|
||||
echo "fail scp-download-failed" >> "$marker"
|
||||
return
|
||||
fi
|
||||
local local_name local_path
|
||||
local_name="$(basename "$short")"
|
||||
local_path="$LOG_DIR/$local_name"
|
||||
if [ ! -s "$local_path" ]; then
|
||||
echo "fail local-empty" >> "$marker"
|
||||
return
|
||||
fi
|
||||
echo "ok $local_name $(wc -c < "$local_path") $(wc -l < "$local_path")" >> "$marker"
|
||||
}
|
||||
|
||||
# ---- launch all workers in parallel --------------------------------------
|
||||
log "launching ${#HOSTS[@]} hosts in parallel..."
|
||||
for host in "${HOSTS[@]}"; do
|
||||
rm -f "$LOG_DIR/_marker.$host"
|
||||
worker "$host" &
|
||||
HOST_PID[$host]=$!
|
||||
HOST_MARKER[$host]="$LOG_DIR/_marker.$host"
|
||||
log " launched $host (pid ${HOST_PID[$host]})"
|
||||
done
|
||||
|
||||
# ---- wait for all, with periodic progress --------------------------------
|
||||
remaining=("${HOSTS[@]}")
|
||||
while [ "${#remaining[@]}" -gt 0 ]; do
|
||||
sleep 10
|
||||
new_remaining=()
|
||||
for host in "${remaining[@]}"; do
|
||||
if ! kill -0 "${HOST_PID[$host]}" 2>/dev/null; then
|
||||
# process finished
|
||||
wait "${HOST_PID[$host]}" 2>/dev/null || true
|
||||
marker="${HOST_MARKER[$host]}"
|
||||
if [ -r "$marker" ]; then
|
||||
status_line="$(tail -n1 "$marker")"
|
||||
log "[$host] done: $status_line"
|
||||
case "$status_line" in
|
||||
ok*) summary_pass+=("$host:$status_line") ;;
|
||||
fail*) summary_fail+=("$host:$status_line") ;;
|
||||
*) summary_fail+=("$host:unknown") ;;
|
||||
esac
|
||||
else
|
||||
log "[$host] done but marker missing"
|
||||
summary_fail+=("$host:no-marker")
|
||||
fi
|
||||
else
|
||||
new_remaining+=("$host")
|
||||
fi
|
||||
done
|
||||
if [ "${#new_remaining[@]}" -gt 0 ]; then
|
||||
remaining=("${new_remaining[@]}")
|
||||
else
|
||||
remaining=()
|
||||
fi
|
||||
if [ "${#remaining[@]}" -gt 0 ]; then
|
||||
log "still running: ${remaining[*]} (${#remaining[@]} hosts)"
|
||||
fi
|
||||
done
|
||||
|
||||
# Final summary
|
||||
log "============================================================"
|
||||
log "DEPLOY SUMMARY"
|
||||
log "============================================================"
|
||||
log "Passed (${#summary_pass[@]}):"
|
||||
for p in "${summary_pass[@]:-}"; do [ -n "$p" ] && log " ✓ $p"; done
|
||||
log "Failed (${#summary_fail[@]}):"
|
||||
for f in "${summary_fail[@]:-}"; do [ -n "$f" ] && log " ✗ $f"; done
|
||||
log ""
|
||||
log "Contents of $LOG_DIR:"
|
||||
ls -la "$LOG_DIR"
|
||||
|
||||
# Clean up marker files
|
||||
rm -f "$LOG_DIR"/_marker.* 2>/dev/null
|
||||
Executable
+56
@@ -0,0 +1,56 @@
|
||||
#!/bin/bash
|
||||
# deploy-tuning.sh - copies apply-tunings.sh to target hosts and runs it.
|
||||
# Usage: bash deploy-tuning.sh [--no-nfs] [--apply] <host> [host...]
|
||||
# Default mode is dry-run. Pass --apply to commit. Pass --no-nfs to skip NFS section.
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/scripts" && pwd)/apply-tunings.sh"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
MODE=""
|
||||
EXTRA_FLAGS=""
|
||||
|
||||
HOSTS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--apply) MODE="--apply" ;;
|
||||
--dry-run) MODE="" ;;
|
||||
--no-nfs) EXTRA_FLAGS="--no-nfs" ;;
|
||||
*) HOSTS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${#HOSTS[@]}" -eq 0 ]; then
|
||||
echo "Usage: $0 <host> [host...] [--apply]"
|
||||
echo "Default: dry-run. Pass --apply to commit."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -r "$SCRIPT" ]; then
|
||||
echo "FATAL: $SCRIPT not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for host in "${HOSTS[@]}"; do
|
||||
echo "================================================================"
|
||||
echo "[$host] deploying apply-tunings.sh (mode: ${MODE:-dry-run})"
|
||||
echo "================================================================"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "[$host] SKIP: unreachable"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[$host] uploading..."
|
||||
if ! scp "${SSH_OPTS[@]}" "$SCRIPT" "root@$host:/root/apply-tunings.sh" >/dev/null 2>&1; then
|
||||
echo "[$host] SKIP: scp failed"
|
||||
continue
|
||||
fi
|
||||
|
||||
echo "[$host] running (output below)..."
|
||||
echo "----------------------------------------------------------------"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" "chmod +x /root/apply-tunings.sh && bash /root/apply-tunings.sh $MODE $EXTRA_FLAGS" 2>&1
|
||||
rc=$?
|
||||
echo "----------------------------------------------------------------"
|
||||
echo "[$host] exit code: $rc"
|
||||
echo ""
|
||||
done
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
# diag.sh - diagnostic commands run on a host via SSH wrapper.
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
echo "===== 1. storage.cfg NFS stanzas (exact content) ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'grep -A 8 "^nfs: D2" /etc/pve/storage.cfg'
|
||||
|
||||
echo ""
|
||||
echo "===== 2. Try manual NFS mount with nconnect=4 ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'mount -t nfs -o nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,proto=tcp pfv-tsys4-nfs-stor:/mnt/tsys4/D2 /mnt/pve/D2 2>&1; echo "exit=$?"'
|
||||
|
||||
echo ""
|
||||
echo "===== 3. Try manual NFS mount WITHOUT nconnect ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'mount -t nfs -o noatime,rsize=1048576,wsize=1048576,hard,proto=tcp pfv-tsys4-nfs-stor:/mnt/tsys4/D2 /mnt/pve/D2 2>&1; echo "exit=$?"'
|
||||
|
||||
echo ""
|
||||
echo "===== 4. NFS kernel version / module ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/fs/nfsfs/version 2>/dev/null; echo "---"; modinfo nfs 2>/dev/null | grep -E "^(filename|version|description)" | head -5'
|
||||
|
||||
echo ""
|
||||
echo "===== 5. mount.nfs version ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'mount.nfs --version 2>&1; echo "---"; dpkg -l nfs-common 2>/dev/null | tail -2'
|
||||
|
||||
echo ""
|
||||
echo "===== 6. /etc/network/interfaces bond0 stanza (exact bytes) ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sed -n "/^auto bond0/,/^$/p" /etc/network/interfaces | cat -A'
|
||||
|
||||
echo ""
|
||||
echo "===== 7. Current bond0 running hash policy ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/net/bonding/bond0 | head -5'
|
||||
|
||||
echo ""
|
||||
echo "===== 8. xmit_hash_policy sysfs file ====="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/class/net/bond0/bonding/xmit_hash_policy 2>/dev/null; echo "---"; ls /sys/class/net/bond0/bonding/ 2>/dev/null'
|
||||
Executable
+128
@@ -0,0 +1,128 @@
|
||||
#!/bin/bash
|
||||
# finish-host.sh - applies ALL remaining changes to a host and verifies.
|
||||
#
|
||||
# Steps:
|
||||
# 1. Start all VMs (triggers NFS lazy-mount)
|
||||
# 2. Wait for NFS mounts to appear
|
||||
# 3. Verify NFS nconnect=4 + noatime
|
||||
# 4. Apply bond0 xmit_hash_policy=layer3+4
|
||||
# 5. Full end-to-end verification
|
||||
#
|
||||
# Usage: bash finish-host.sh <host> [--apply]
|
||||
# Default is dry-run (starts VMs + shows what bond change would do, but doesn't edit interfaces)
|
||||
set -uo pipefail
|
||||
|
||||
HOST="${1:-}"
|
||||
MODE="${2:-dryrun}"
|
||||
[ "$MODE" = "--apply" ] && MODE="apply" || MODE="dryrun"
|
||||
|
||||
if [ -z "$HOST" ]; then
|
||||
echo "Usage: $0 <host> [--apply]"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scripts"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " finish-host.sh — $HOST (mode: $MODE)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# =========================================================================
|
||||
# STEP 1: Start all VMs
|
||||
# =========================================================================
|
||||
echo "=== STEP 1: Start all VMs on $HOST ==="
|
||||
# Get list of all VMs (not just stopped — start is idempotent)
|
||||
vm_list=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1{print \$1}"')
|
||||
for vmid in $vm_list; do
|
||||
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
||||
if [ "$status" != "running" ]; then
|
||||
echo " Starting VM $vmid..."
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1 | sed 's/^/ /'
|
||||
else
|
||||
echo " VM $vmid already running"
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo " Waiting 15s for VMs to boot and trigger NFS mounts..."
|
||||
sleep 15
|
||||
|
||||
echo ""
|
||||
echo "--- VM status after start ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
|
||||
|
||||
# =========================================================================
|
||||
# STEP 2: Verify NFS mounts came back with nconnect=4
|
||||
# =========================================================================
|
||||
echo ""
|
||||
echo "=== STEP 2: Verify NFS mounts with nconnect=4 ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null' | head -30
|
||||
|
||||
echo ""
|
||||
echo "--- NFS TCP connections to :2049 ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||||
conn_count=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l')
|
||||
echo " Total NFS TCP connections: $conn_count"
|
||||
|
||||
# =========================================================================
|
||||
# STEP 3: Apply bond0 hash policy
|
||||
# =========================================================================
|
||||
echo ""
|
||||
echo "=== STEP 3: Apply bond0 xmit_hash_policy=layer3+4 (mode: $MODE) ==="
|
||||
|
||||
# Upload the bond hash script
|
||||
scp "${SSH_OPTS[@]}" "$SCRIPT_DIR/apply-bond-hash.sh" "root@$HOST:/root/apply-bond-hash.sh" >/dev/null 2>&1
|
||||
|
||||
if [ "$MODE" = "apply" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'chmod +x /root/apply-bond-hash.sh && bash /root/apply-bond-hash.sh --apply' 2>&1
|
||||
else
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'chmod +x /root/apply-bond-hash.sh && bash /root/apply-bond-hash.sh' 2>&1
|
||||
fi
|
||||
|
||||
# =========================================================================
|
||||
# STEP 4: Full verification
|
||||
# =========================================================================
|
||||
echo ""
|
||||
echo "=== STEP 4: Full end-to-end verification ==="
|
||||
echo ""
|
||||
echo "--- Uptime ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime'
|
||||
echo ""
|
||||
echo "--- CPU governor ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "(no cpufreq driver)"'
|
||||
echo ""
|
||||
echo "--- vm.swappiness ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl vm.swappiness'
|
||||
echo ""
|
||||
echo "--- TCP BBR ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc'
|
||||
echo ""
|
||||
echo "--- tuned profile ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'tuned-adm active 2>/dev/null'
|
||||
echo ""
|
||||
echo "--- bond0 hash policy + LACP state ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/net/bonding/bond0 2>/dev/null | head -25'
|
||||
echo ""
|
||||
echo "--- NFS mount options (first 3 mounts) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -24'
|
||||
echo ""
|
||||
echo "--- NFS TCP connections (expect 4 per server × 2 servers = 8) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||||
nfs_conns=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l')
|
||||
echo " Count: $nfs_conns"
|
||||
echo ""
|
||||
echo "--- VMs running ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
|
||||
echo ""
|
||||
echo "--- Failed services ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'systemctl --failed --no-legend 2>/dev/null | head -10'
|
||||
echo ""
|
||||
echo "--- Network interfaces (speed/duplex/mtu) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'for ifc in bond0 nic0 nic1 nic2 vmbr0 datanet; do [ -d "/sys/class/net/$ifc" ] && printf "%-12s speed=%-8s duplex=%-8s mtu=%s\n" "$ifc" "$(cat /sys/class/net/$ifc/speed 2>/dev/null)" "$(cat /sys/class/net/$ifc/duplex 2>/dev/null)" "$(cat /sys/class/net/$ifc/mtu 2>/dev/null)"; done'
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " COMPLETE — $HOST"
|
||||
echo "==================================================================="
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
# install-utils-v2.sh - retry install without nstat package.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
PKGS="sysstat jq numactl nvme-cli mtr-tiny dnsutils bmon"
|
||||
|
||||
for host in pfv-tsys6 pfv-tsys7; do
|
||||
echo "=== [$host] installing: $PKGS ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>&1 | tail -2 && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y $PKGS 2>&1 | tail -10"
|
||||
|
||||
# sysstat enable (path varies by Debian version)
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" '
|
||||
if [ -r /etc/default/sysstat ]; then
|
||||
sed -i "s/^ENABLED=.*/ENABLED=\"true\"/" /etc/default/sysstat
|
||||
systemctl enable --now sysstat 2>/dev/null
|
||||
grep ENABLED /etc/default/sysstat
|
||||
else
|
||||
# Newer Debian (trixie) — sysstat cron/service auto-enabled
|
||||
systemctl enable --now sysstat 2>/dev/null || echo "(sysstat auto via cron)"
|
||||
fi
|
||||
'
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Verify
|
||||
for host in pfv-tsys6 pfv-tsys7; do
|
||||
echo "=== [$host] verification ==="
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" '
|
||||
for p in sysstat jq numactl nvme mtr-tiny dig bmon; do
|
||||
command -v "$p" >/dev/null 2>&1 && echo " ✓ $p" || echo " ✗ $p"
|
||||
done
|
||||
'
|
||||
echo ""
|
||||
done
|
||||
Executable
+60
@@ -0,0 +1,60 @@
|
||||
#!/bin/bash
|
||||
# install-utils.sh - installs useful observability packages on a host.
|
||||
# These are all small, dependency-light, and read-only at runtime.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
HOSTS=(pfv-tsys6 pfv-tsys7)
|
||||
|
||||
# Packages to install, with rationale
|
||||
PKGS=(
|
||||
sysstat # sar, iostat, mpstat, pidstat - the missing observability suite
|
||||
jq # JSON parsing for pvesh/scripts
|
||||
numactl # NUMA topology/controls for the dual-socket hosts
|
||||
nvme-cli # NVMe health (for when NVMe shows up)
|
||||
tcpdump # packet capture for network debugging
|
||||
mtr-tiny # traceroute on steroids
|
||||
nstat # kernel SNMP stats (already partly there)
|
||||
dnsutils # dig, nslookup, host
|
||||
bmon # bandwidth monitor ( curses, real-time)
|
||||
)
|
||||
|
||||
for host in "${HOSTS[@]}"; do
|
||||
echo "================================================================"
|
||||
echo "[$host] installing observability packages"
|
||||
echo "================================================================"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Check which are missing
|
||||
missing=""
|
||||
for pkg in "${PKGS[@]}"; do
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" "dpkg -s $pkg 2>/dev/null | grep -q 'Status: install ok installed'" 2>/dev/null; then
|
||||
missing="$missing $pkg"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ -z "$missing" ]; then
|
||||
echo " All packages already installed."
|
||||
continue
|
||||
fi
|
||||
|
||||
echo " Installing:$missing"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq$missing 2>&1 | tail -5"
|
||||
echo " Done."
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Enable sysstat data collection (sar) — off by default on Debian
|
||||
for host in "${HOSTS[@]}"; do
|
||||
echo "[$host] enabling sysstat/sar data collection..."
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
"sed -i 's/^ENABLED=\"false\"/ENABLED=\"true\"/' /etc/default/sysstat 2>/dev/null; \
|
||||
systemctl enable --now sysstat 2>&1 | tail -2; \
|
||||
grep ENABLED /etc/default/sysstat"
|
||||
done
|
||||
Executable
+293
@@ -0,0 +1,293 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# iperf-full-matrix.sh
|
||||
#
|
||||
# Two test suites:
|
||||
# A. Management network (vmbr0 / VLAN1): all-pairs single-stream TCP, 10s
|
||||
# B. Storage network (VLAN1000): tsys6+tsys7 → tsys4+tsys5, stress test
|
||||
#
|
||||
# Output: returned-logs/iperf/
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/returned-logs/iperf"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7)
|
||||
|
||||
# Storage IPs (known, static on VLAN1000)
|
||||
declare -A SIP
|
||||
SIP[pfv-tsys1]="10.100.100.1"
|
||||
SIP[pfv-tsys3]="10.100.100.3"
|
||||
SIP[pfv-tsys4]="10.100.100.4"
|
||||
SIP[pfv-tsys5]="10.100.100.5"
|
||||
SIP[pfv-tsys6]="10.100.100.6"
|
||||
SIP[pfv-tsys7]="10.100.100.7"
|
||||
|
||||
# ===========================================================================
|
||||
# STEP 0: Discover management IPs (vmbr0)
|
||||
# ===========================================================================
|
||||
echo "==================================================================="
|
||||
echo " STEP 0: Discover management network IPs (vmbr0)"
|
||||
echo "==================================================================="
|
||||
declare -A MIP
|
||||
for host in "${HOSTS[@]}"; do
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " [$host] UNREACHABLE"
|
||||
continue
|
||||
fi
|
||||
mip=$(ssh "${SSH_OPTS[@]}" "root@$host" 'ip -o -4 addr show dev vmbr0 2>/dev/null | awk "{print \$4}" | cut -d/ -f1 | head -1')
|
||||
if [ -n "$mip" ]; then
|
||||
MIP[$host]="$mip"
|
||||
echo " [$host] vmbr0 = $mip"
|
||||
else
|
||||
echo " [$host] no vmbr0 IPv4 — skipping"
|
||||
fi
|
||||
done
|
||||
|
||||
# ===========================================================================
|
||||
# STEP 1: Ensure iperf3 installed on all hosts
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " STEP 1: Ensure iperf3 installed"
|
||||
echo "==================================================================="
|
||||
for host in "${HOSTS[@]}"; do
|
||||
[ -z "${MIP[$host]:-}" ] && continue
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'command -v iperf3 >/dev/null 2>&1' 2>/dev/null; then
|
||||
echo -n " [$host] installing iperf3... "
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
'DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && \
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -qq iperf3 >/dev/null 2>&1 && echo OK || echo FAILED'
|
||||
else
|
||||
echo " [$host] iperf3 already installed"
|
||||
fi
|
||||
done
|
||||
|
||||
# Helper: start iperf3 server in one-shot mode bound to a specific IP
|
||||
start_server() {
|
||||
local host="$1" ip="$2"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" \
|
||||
"pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${ip} >/dev/null 2>&1 &" 2>/dev/null
|
||||
sleep 1
|
||||
}
|
||||
|
||||
# Helper: run iperf3 client test, save output, extract result
|
||||
run_test() {
|
||||
local client="$1" server="$2" sip="$3" cip="$4" label="$5" logfile="$6"
|
||||
shift 6
|
||||
local extra="$*"
|
||||
|
||||
echo -n " [$label] ... "
|
||||
{
|
||||
echo "=== iperf3: $label ==="
|
||||
echo "Client: $client ($cip) → Server: $server ($sip)"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Args: $extra"
|
||||
echo ""
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" "iperf3 -c ${sip} -B ${cip} ${extra}" 2>&1
|
||||
echo ""
|
||||
echo "=== END ==="
|
||||
} > "$logfile" 2>&1
|
||||
|
||||
# Extract result
|
||||
sum=$(grep '\[SUM\].*sender$' "$logfile" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
else
|
||||
single=$(grep 'sender$' "$logfile" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
fi
|
||||
echo "${bitrate:-?} (retrans: ${retrans:-?})"
|
||||
}
|
||||
|
||||
# ===========================================================================
|
||||
# SUITE A: Management network (vmbr0) — all pairs, single-stream TCP, 10s
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SUITE A: MANAGEMENT NETWORK (vmbr0) — all-pairs, 10s TCP"
|
||||
echo " Expectation: ~940 Mbps for every pair (1 GbE line rate)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# Test each unique pair (i < j to avoid duplicates)
|
||||
for ((i=0; i<${#HOSTS[@]}; i++)); do
|
||||
for ((j=i+1; j<${#HOSTS[@]}; j++)); do
|
||||
client="${HOSTS[$i]}"
|
||||
server="${HOSTS[$j]}"
|
||||
[ -z "${MIP[$client]:-}" ] && continue
|
||||
[ -z "${MIP[$server]:-}" ] && continue
|
||||
|
||||
label="mgmt: ${client}→${server}"
|
||||
logfile="$LOG_DIR/mgmt-${client}-to-${server}.log"
|
||||
|
||||
start_server "$server" "${MIP[$server]}"
|
||||
run_test "$client" "$server" "${MIP[$server]}" "${MIP[$client]}" \
|
||||
"$label" "$logfile" "-t 10 -P 1"
|
||||
done
|
||||
done
|
||||
|
||||
# ===========================================================================
|
||||
# SUITE B: Storage network (VLAN1000) — stress test the cross-rack LACP
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SUITE B: STORAGE NETWORK (VLAN1000) — stress test cross-rack link"
|
||||
echo " tsys6 + tsys7 (Rack 3) → tsys4 + tsys5 (Rack 5)"
|
||||
echo " Expectation: limited by tsys4 USB dongle + tsys5 broken bond"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# --- B.1: Individual tests (one client → one server at a time) ---
|
||||
echo "--- B.1: Individual tests (sequential) ---"
|
||||
echo ""
|
||||
|
||||
for client in pfv-tsys6 pfv-tsys7; do
|
||||
for server in pfv-tsys4 pfv-tsys5; do
|
||||
label="stor: ${client}→${server} (8-stream)"
|
||||
logfile="$LOG_DIR/stor-indiv-${client}-to-${server}-8stream.log"
|
||||
start_server "$server" "${SIP[$server]}"
|
||||
run_test "$client" "$server" "${SIP[$server]}" "${SIP[$client]}" \
|
||||
"$label" "$logfile" "-P 8 -t 20 -l 128k -O 2"
|
||||
done
|
||||
done
|
||||
|
||||
# --- B.2: Reverse direction (tsys4/5 → tsys6/7) ---
|
||||
echo ""
|
||||
echo "--- B.2: Reverse direction (tsys4/5 → tsys6/7) ---"
|
||||
echo ""
|
||||
|
||||
for client in pfv-tsys4 pfv-tsys5; do
|
||||
for server in pfv-tsys6 pfv-tsys7; do
|
||||
label="stor: ${client}→${server} (8-stream rev)"
|
||||
logfile="$LOG_DIR/stor-indiv-${client}-to-${server}-8stream.log"
|
||||
start_server "$server" "${SIP[$server]}"
|
||||
run_test "$client" "$server" "${SIP[$server]}" "${SIP[$client]}" \
|
||||
"$label" "$logfile" "-P 8 -t 20 -l 128k -O 2"
|
||||
done
|
||||
done
|
||||
|
||||
# --- B.3: Simultaneous stress test (4 flows at once) ---
|
||||
echo ""
|
||||
echo "--- B.3: Simultaneous 4-flow stress test ---"
|
||||
echo " tsys6→tsys4 + tsys6→tsys5 + tsys7→tsys4 + tsys7→tsys5"
|
||||
echo " All running in parallel for 30 seconds"
|
||||
echo ""
|
||||
|
||||
# Start 4 iperf3 servers (one-shot mode won't work for parallel; use persistent)
|
||||
for server in pfv-tsys4 pfv-tsys5; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -B ${SIP[$server]} >/dev/null 2>&1 &" 2>/dev/null
|
||||
echo " [server started: $server]"
|
||||
done
|
||||
sleep 1
|
||||
|
||||
STRESS_LOG="$LOG_DIR/stor-stress-4flow"
|
||||
mkdir -p "$STRESS_LOG"
|
||||
|
||||
# Launch 4 clients in parallel, each writing to its own log
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" "iperf3 -c ${SIP[pfv-tsys4]} -B ${SIP[pfv-tsys6]} -P 4 -t 30 -l 128k -O 2" > "$STRESS_LOG/tsys6-to-tsys4.log" 2>&1 &
|
||||
PID1=$!
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" "iperf3 -c ${SIP[pfv-tsys5]} -B ${SIP[pfv-tsys6]} -P 4 -t 30 -l 128k -O 2" > "$STRESS_LOG/tsys6-to-tsys5.log" 2>&1 &
|
||||
PID2=$!
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" "iperf3 -c ${SIP[pfv-tsys4]} -B ${SIP[pfv-tsys7]} -P 4 -t 30 -l 128k -O 2" > "$STRESS_LOG/tsys7-to-tsys4.log" 2>&1 &
|
||||
PID3=$!
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" "iperf3 -c ${SIP[pfv-tsys5]} -B ${SIP[pfv-tsys7]} -P 4 -t 30 -l 128k -O 2" > "$STRESS_LOG/tsys7-to-tsys5.log" 2>&1 &
|
||||
PID4=$!
|
||||
|
||||
echo " [4 clients launched, waiting 40s for completion...]"
|
||||
wait $PID1 $PID2 $PID3 $PID4 2>/dev/null
|
||||
echo " [all 4 flows complete]"
|
||||
|
||||
# Kill servers
|
||||
for server in pfv-tsys4 pfv-tsys5; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$server" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
|
||||
# ===========================================================================
|
||||
# SUITE C: All hosts cleanup
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Cleanup: killing iperf3 everywhere"
|
||||
echo "==================================================================="
|
||||
for host in "${HOSTS[@]}"; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
|
||||
# ===========================================================================
|
||||
# RESULTS SUMMARY
|
||||
# ===========================================================================
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " RESULTS SUMMARY"
|
||||
echo "==================================================================="
|
||||
|
||||
echo ""
|
||||
echo "===== SUITE A: Management network (vmbr0) ====="
|
||||
echo ""
|
||||
printf "%-40s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
|
||||
printf "%-40s %15s %10s\n" "----" "----------" "-------"
|
||||
for f in "$LOG_DIR"/mgmt-*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||||
single=$(grep 'sender$' "$f" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
printf "%-40s %15s %10s\n" "$label" "${bitrate:-?}" "${retrans:--}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "===== SUITE B.1+B.2: Storage network individual ====="
|
||||
echo ""
|
||||
printf "%-45s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
|
||||
printf "%-45s %15s %10s\n" "----" "----------" "-------"
|
||||
for f in "$LOG_DIR"/stor-indiv-*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||||
sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
else
|
||||
single=$(grep 'sender$' "$f" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
fi
|
||||
printf "%-45s %15s %10s\n" "$label" "${bitrate:-?}" "${retrans:--}"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "===== SUITE B.3: Simultaneous 4-flow stress test ====="
|
||||
echo ""
|
||||
printf "%-30s %15s %10s\n" "FLOW" "THROUGHPUT" "RETRANS"
|
||||
printf "%-30s %15s %10s\n" "----" "----------" "-------"
|
||||
total_mbps=0
|
||||
for f in "$STRESS_LOG"/*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
flow=$(basename "$f" .log)
|
||||
sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
mbps=$(echo "$sum" | awk '{print $6}')
|
||||
total_mbps=$(awk "BEGIN{print $total_mbps + $mbps}")
|
||||
else
|
||||
bitrate="?"
|
||||
retrans="-"
|
||||
fi
|
||||
printf "%-30s %15s %10s\n" "$flow" "$bitrate" "${retrans:--}"
|
||||
done
|
||||
printf "%-30s %15s\n" "AGGREGATE (all 4 flows)" "${total_mbps} Mbits/sec"
|
||||
|
||||
echo ""
|
||||
echo "===== CONTEXT ====="
|
||||
echo "tsys4: USB cdc_ncm dongle (single 1G link, no bond)"
|
||||
echo "tsys5: bond0 broken (1 active slave, no LACP partner) — cable pending"
|
||||
echo "tsys6/7: working 2x1G LACP, layer3+4 hash (host side)"
|
||||
echo "Cross-rack: 4x1G LACP (pfv-r3-tor-stor → pfv-core-sw01)"
|
||||
echo ""
|
||||
echo "All logs in: $LOG_DIR/"
|
||||
Executable
+184
@@ -0,0 +1,184 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# iperf-storage-tests.sh
|
||||
#
|
||||
# Installs iperf3 on all online hosts, then runs a matrix of storage-network
|
||||
# throughput tests. Saves all output to returned-logs/iperf/.
|
||||
#
|
||||
# Test matrix (all over VLAN1000 storage network, 10.100.100.0/24):
|
||||
# 1. tsys7 → tsys4 (USB cdc_ncm NIC) — the smoking gun
|
||||
# 2. tsys7 → tsys5 (bond0, 1 active slave) — PCI NIC comparison
|
||||
# 3. tsys7 → tsys6 (bond0, 2 active slaves) — working LACP baseline
|
||||
# 4. Reverse: tsys4 → tsys7 (USB NIC TX direction)
|
||||
# 5. Reverse: tsys5 → tsys7
|
||||
#
|
||||
# Each test: TCP 8-stream 30s forward + reverse + UDP saturation.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/returned-logs/iperf"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
ALL_HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys4 pfv-tsys5 pfv-tsys6 pfv-tsys7)
|
||||
|
||||
# Storage network IPs
|
||||
declare -A SIP
|
||||
SIP[pfv-tsys1]="10.100.100.1"
|
||||
SIP[pfv-tsys3]="10.100.100.3"
|
||||
SIP[pfv-tsys4]="10.100.100.4"
|
||||
SIP[pfv-tsys5]="10.100.100.5"
|
||||
SIP[pfv-tsys6]="10.100.100.6"
|
||||
SIP[pfv-tsys7]="10.100.100.7"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " STEP 1: Install iperf3 on all online hosts"
|
||||
echo "==================================================================="
|
||||
|
||||
for host in "${ALL_HOSTS[@]}"; do
|
||||
echo -n " [$host] "
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$host" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "UNREACHABLE — skipping"
|
||||
continue
|
||||
fi
|
||||
# Check if iperf3 already installed
|
||||
if ssh "${SSH_OPTS[@]}" "root@$host" 'command -v iperf3 >/dev/null 2>&1' 2>/dev/null; then
|
||||
echo "iperf3 already installed"
|
||||
else
|
||||
printf "installing... "
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq iperf3 >/dev/null 2>&1 && echo OK || echo FAILED'
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " STEP 2: Kill any existing iperf3 processes everywhere"
|
||||
echo "==================================================================="
|
||||
for host in "${ALL_HOSTS[@]}"; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo " Done."
|
||||
|
||||
# Helper: run an iperf3 test and save output
|
||||
run_iperf() {
|
||||
local client="$1" server="$2" direction="$3" label="$4" logfile="$5"
|
||||
local client_ip="${SIP[$client]}" server_ip="${SIP[$server]}"
|
||||
|
||||
echo -n " [$client → $server] $label ... "
|
||||
|
||||
# Start server in one-shot mode (-1 means serve one client then exit)
|
||||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
|
||||
sleep 1
|
||||
|
||||
# Run client
|
||||
{
|
||||
echo "=== iperf3: $label ==="
|
||||
echo "Client: $client ($client_ip)"
|
||||
echo "Server: $server ($server_ip)"
|
||||
echo "Direction: $direction"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo ""
|
||||
if [ "$direction" = "forward" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||||
"iperf3 -c ${server_ip} -B ${client_ip} -P 8 -t 30 -l 128k -O 2" 2>&1
|
||||
elif [ "$direction" = "reverse" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||||
"iperf3 -c ${server_ip} -B ${client_ip} -P 8 -t 30 -l 128k -O 2 -R" 2>&1
|
||||
elif [ "$direction" = "udp" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||||
"iperf3 -c ${server_ip} -B ${client_ip} -u -b 2G -t 10 -l 8972" 2>&1
|
||||
elif [ "$direction" = "single" ]; then
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" \
|
||||
"iperf3 -c ${server_ip} -B ${client_ip} -t 20 -O 2" 2>&1
|
||||
fi
|
||||
echo ""
|
||||
echo "=== END ==="
|
||||
} > "$logfile" 2>&1
|
||||
|
||||
# Extract summary line
|
||||
if grep -q "sender" "$logfile"; then
|
||||
bitrate=$(grep "sender" "$logfile" | tail -1 | awk '{print $7, $8}')
|
||||
echo "done: ${bitrate}"
|
||||
else
|
||||
echo "done (check log for details)"
|
||||
fi
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " STEP 3: Run iperf3 test matrix"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
echo "All tests over VLAN1000 storage network (10.100.100.0/24)."
|
||||
echo "TCP tests: 8 parallel streams, 30s, 128k blocks."
|
||||
echo ""
|
||||
|
||||
# --- Test 1: tsys7 → tsys4 (USB cdc_ncm target) ---
|
||||
echo "--- TEST 1: tsys7 → tsys4 (USB cdc_ncm NIC) ---"
|
||||
run_iperf pfv-tsys7 pfv-tsys4 forward "TCP 8-stream forward (tsys7→tsys4 USB)" \
|
||||
"$LOG_DIR/01-tsys7-to-tsys4-tcp-forward.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys4 reverse "TCP 8-stream reverse (tsys4 USB→tsys7)" \
|
||||
"$LOG_DIR/02-tsys7-to-tsys4-tcp-reverse.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys4 single "TCP single-stream forward (tsys7→tsys4 USB)" \
|
||||
"$LOG_DIR/03-tsys7-to-tsys4-tcp-single.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys4 udp "UDP saturation (tsys7→tsys4 USB)" \
|
||||
"$LOG_DIR/04-tsys7-to-tsys4-udp.log"
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Test 2: tsys7 → tsys5 (bond0, PCI NIC, 1 active slave) ---
|
||||
echo "--- TEST 2: tsys7 → tsys5 (PCI NIC, broken bond - 1 slave) ---"
|
||||
run_iperf pfv-tsys7 pfv-tsys5 forward "TCP 8-stream forward (tsys7→tsys5 PCI)" \
|
||||
"$LOG_DIR/05-tsys7-to-tsys5-tcp-forward.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys5 reverse "TCP 8-stream reverse (tsys5 PCI→tsys7)" \
|
||||
"$LOG_DIR/06-tsys7-to-tsys5-tcp-reverse.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys5 single "TCP single-stream forward (tsys7→tsys5 PCI)" \
|
||||
"$LOG_DIR/07-tsys7-to-tsys5-tcp-single.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys5 udp "UDP saturation (tsys7→tsys5 PCI)" \
|
||||
"$LOG_DIR/08-tsys7-to-tsys5-udp.log"
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Test 3: tsys7 → tsys6 (working 2-slave LACP baseline, layer3+4) ---
|
||||
echo "--- TEST 3: tsys7 → tsys6 (working 2×1G LACP baseline) ---"
|
||||
run_iperf pfv-tsys7 pfv-tsys6 forward "TCP 8-stream forward (tsys7→tsys6 LACP)" \
|
||||
"$LOG_DIR/09-tsys7-to-tsys6-tcp-forward.log"
|
||||
run_iperf pfv-tsys7 pfv-tsys6 reverse "TCP 8-stream reverse (tsys6 LACP→tsys7)" \
|
||||
"$LOG_DIR/10-tsys7-to-tsys6-tcp-reverse.log"
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Test 4: tsys6 → tsys4 (pre-tuning baseline) ---
|
||||
echo "--- TEST 4: tsys6 → tsys4 (baseline before tsys6 tuning) ---"
|
||||
run_iperf pfv-tsys6 pfv-tsys4 forward "TCP 8-stream forward (tsys6→tsys4 USB)" \
|
||||
"$LOG_DIR/11-tsys6-to-tsys4-tcp-forward.log"
|
||||
run_iperf pfv-tsys6 pfv-tsys4 reverse "TCP 8-stream reverse (tsys4 USB→tsys6)" \
|
||||
"$LOG_DIR/12-tsys6-to-tsys4-tcp-reverse.log"
|
||||
|
||||
echo ""
|
||||
|
||||
# --- Cleanup: kill iperf3 everywhere ---
|
||||
echo "--- Cleanup ---"
|
||||
for host in "${ALL_HOSTS[@]}"; do
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " RESULTS SUMMARY"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
printf "%-45s %s\n" "TEST" "THROUGHPUT"
|
||||
printf "%-45s %s\n" "----" "----------"
|
||||
for f in "$LOG_DIR"/*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||||
bitrate=$(grep -E "sender$" "$f" | tail -1 | awk '{print $7, $8}')
|
||||
[ -z "$bitrate" ] && bitrate=$(grep -E "Mbits/sec|Gbits/sec" "$f" | tail -1 | grep -oE '[0-9.]+ [MG]bits/sec' | head -1)
|
||||
[ -z "$bitrate" ] && bitrate="(see log)"
|
||||
printf "%-45s %s\n" "$label" "$bitrate"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "Full logs saved to: $LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
Executable
+119
@@ -0,0 +1,119 @@
|
||||
#!/bin/bash
|
||||
# iperf-tsys6-tsys7.sh - validate 2Gbps LACP between the two tuned hosts.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/returned-logs/iperf"
|
||||
mkdir -p "$LOG_DIR"
|
||||
|
||||
TSYS6="10.100.100.6"
|
||||
TSYS7="10.100.100.7"
|
||||
|
||||
run_test() {
|
||||
local client="$1" server="$2" server_ip="$3" label="$4" logfile="$5"
|
||||
shift 4
|
||||
local extra_args="$*"
|
||||
|
||||
echo -n " [$label] ... "
|
||||
|
||||
# Start server in one-shot mode
|
||||
ssh "${SSH_OPTS[@]}" "root@$server" "pkill -x iperf3 2>/dev/null; nohup iperf3 -s -1 -B ${server_ip} >/dev/null 2>&1 &" 2>/dev/null
|
||||
sleep 1
|
||||
|
||||
{
|
||||
echo "=== iperf3: $label ==="
|
||||
echo "Client: $client Server: $server ($server_ip)"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Args: $extra_args"
|
||||
echo ""
|
||||
ssh "${SSH_OPTS[@]}" "root@$client" "iperf3 -c ${server_ip} $extra_args" 2>&1
|
||||
echo ""
|
||||
echo "=== END ==="
|
||||
} > "$logfile" 2>&1
|
||||
|
||||
# Extract result
|
||||
sum=$(grep '\[SUM\].*sender$' "$logfile" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
else
|
||||
single=$(grep 'sender$' "$logfile" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
fi
|
||||
echo "${bitrate} (retrans: ${retrans:-0})"
|
||||
}
|
||||
|
||||
echo "==================================================================="
|
||||
echo " iperf3: tsys6 ↔ tsys7 (both have 2×1G LACP + layer3+4 hash)"
|
||||
echo " Expectation: ~1.8-2.0 Gbps for 8-stream TCP"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# Pre-flight: confirm bond state on both
|
||||
echo "--- bond0 state on tsys6 ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
|
||||
echo ""
|
||||
echo "--- bond0 state on tsys7 ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'grep -E "Transmit Hash|Number of ports|Bonding Mode" /proc/net/bonding/bond0'
|
||||
echo ""
|
||||
|
||||
echo "--- Running tests ---"
|
||||
echo ""
|
||||
|
||||
# Test 1: tsys7 → tsys6, 8-stream TCP forward
|
||||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||||
"tsys7→tsys6 TCP 8-stream forward" \
|
||||
"$LOG_DIR/tsys6-tsys7-01-tcp-8stream-forward.log" \
|
||||
"-P 8 -t 30 -l 128k -O 2"
|
||||
|
||||
# Test 2: tsys6 → tsys7, 8-stream TCP forward (reverse direction)
|
||||
run_test pfv-tsys6 pfv-tsys7 "$TSYS7" \
|
||||
"tsys6→tsys7 TCP 8-stream forward" \
|
||||
"$LOG_DIR/tsys6-tsys7-02-tcp-8stream-forward.log" \
|
||||
"-P 8 -t 30 -l 128k -O 2"
|
||||
|
||||
# Test 3: tsys7 → tsys6, single stream (should be ~940 Mbps — single flow)
|
||||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||||
"tsys7→tsys6 TCP single-stream" \
|
||||
"$LOG_DIR/tsys6-tsys7-03-tcp-single.log" \
|
||||
"-t 20 -O 2"
|
||||
|
||||
# Test 4: tsys7 → tsys6, 4-stream (nconnect=4 mirrors this)
|
||||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||||
"tsys7→tsys6 TCP 4-stream" \
|
||||
"$LOG_DIR/tsys6-tsys7-04-tcp-4stream.log" \
|
||||
"-P 4 -t 30 -l 128k -O 2"
|
||||
|
||||
# Test 5: UDP saturation
|
||||
run_test pfv-tsys7 pfv-tsys6 "$TSYS6" \
|
||||
"tsys7→tsys6 UDP saturation" \
|
||||
"$LOG_DIR/tsys6-tsys7-05-udp.log" \
|
||||
"-u -b 3G -t 10 -l 8972"
|
||||
|
||||
# Cleanup
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys6" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
ssh "${SSH_OPTS[@]}" "root@pfv-tsys7" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SUMMARY"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
printf "%-45s %15s %10s\n" "TEST" "THROUGHPUT" "RETRANS"
|
||||
printf "%-45s %15s %10s\n" "----" "----------" "-------"
|
||||
for f in "$LOG_DIR"/tsys6-tsys7-*.log; do
|
||||
[ -r "$f" ] || continue
|
||||
label=$(head -1 "$f" | sed 's/^=== iperf3: //; s/ ===$//')
|
||||
sum=$(grep '\[SUM\].*sender$' "$f" | tail -1)
|
||||
if [ -n "$sum" ]; then
|
||||
bitrate=$(echo "$sum" | awk '{print $6, $7}')
|
||||
retrans=$(echo "$sum" | awk '{print $8}')
|
||||
else
|
||||
single=$(grep 'sender$' "$f" | tail -1)
|
||||
bitrate=$(echo "$single" | awk '{print $7, $8}')
|
||||
retrans=$(echo "$single" | awk '{print $9}')
|
||||
fi
|
||||
printf "%-45s %15s %10s\n" "$label" "$bitrate" "${retrans:--}"
|
||||
done
|
||||
echo ""
|
||||
echo "Expected: 8-stream ~1.8-2.0 Gbps, single-stream ~940 Mbps"
|
||||
Executable
+88
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
# reboot-and-verify.sh - reboots a host and verifies NFS nconnect activates.
|
||||
# Usage: bash reboot-and-verify.sh <host>
|
||||
set -uo pipefail
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
echo "================================================================"
|
||||
echo "[$HOST] PRE-REBOOT STATE"
|
||||
echo "================================================================"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
echo "--- VMs ---"
|
||||
qm list 2>/dev/null
|
||||
echo "--- NFS TCP conns: $(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l) ---"
|
||||
echo "--- NFS first mount: ---"
|
||||
nfsstat -m 2>/dev/null | head -2 | tail -1
|
||||
echo "--- uptime ---"
|
||||
uptime
|
||||
'
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] ISSUING REBOOT"
|
||||
echo "================================================================"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nohup sh -c "(sleep 2; systemctl reboot)" >/dev/null 2>&1 &'
|
||||
echo "Reboot sent at $(date +%H:%M:%S)"
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] WAITING FOR SSH TO RETURN (max 10 min)"
|
||||
echo "================================================================"
|
||||
DEADLINE=$(( $(date +%s) + 600 ))
|
||||
LAST_PRINT=0
|
||||
while [ "$(date +%s)" -lt "$DEADLINE" ]; do
|
||||
now=$(date +%s)
|
||||
if [ $((now - LAST_PRINT)) -ge 15 ]; then
|
||||
printf ' [%s] waiting... (%ss elapsed)\n' "$(date +%H:%M:%S)" "$(( now - DEADLINE + 600 ))"
|
||||
LAST_PRINT=$now
|
||||
fi
|
||||
if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
# Verify uptime is actually low (host really rebooted, not still up)
|
||||
up_mins=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /proc/uptime | awk "{print int(\$1/60)}"')
|
||||
if [ "${up_mins:-999}" -lt 5 ]; then
|
||||
echo " [$(date +%H:%M:%S)] SSH back, uptime ${up_mins}min — real reboot confirmed"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "FAILED: $HOST not back after 10 minutes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Waiting 30s for services to settle..."
|
||||
sleep 30
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] POST-REBOOT VERIFICATION"
|
||||
echo "================================================================"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
echo "--- uptime ---"
|
||||
uptime
|
||||
echo ""
|
||||
echo "--- governor: $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null) ---"
|
||||
echo "--- swappiness: $(sysctl -n vm.swappiness) ---"
|
||||
echo "--- tcp_cc: $(sysctl -n net.ipv4.tcp_congestion_control) ---"
|
||||
echo "--- tuned: $(tuned-adm active 2>/dev/null | grep Current) ---"
|
||||
echo ""
|
||||
echo "--- NFS first mount: ---"
|
||||
nfsstat -m 2>/dev/null | head -2 | tail -1
|
||||
echo ""
|
||||
echo "--- NFS TCP conns (expect 8 with nconnect=4): ---"
|
||||
ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l
|
||||
echo ""
|
||||
echo "--- VMs: ---"
|
||||
qm list 2>/dev/null
|
||||
echo ""
|
||||
echo "--- Failed services: ---"
|
||||
systemctl --failed --no-legend 2>/dev/null | head -5
|
||||
echo "(empty = none)"
|
||||
'
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] DONE"
|
||||
echo "================================================================"
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
# reboot-verify.sh - reboots a host, waits for it to come back, verifies state.
|
||||
# Usage: bash reboot-verify.sh <host>
|
||||
set -uo pipefail
|
||||
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
echo "================================================================"
|
||||
echo "[$HOST] PRE-REBOOT STATE"
|
||||
echo "================================================================"
|
||||
echo "--- Running VMs ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR==1 || \$3==\"running\"{print}"' 2>&1
|
||||
echo "--- NFS mount count ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | grep -c "^/mnt"' 2>&1
|
||||
echo "--- NFS TCP connections to :2049 ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l' 2>&1
|
||||
echo "--- Uptime ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime' 2>&1
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] ISSUING REBOOT"
|
||||
echo "================================================================"
|
||||
# Issue reboot; ssh will disconnect with non-zero — that's expected.
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nohup sh -c "(sleep 2; systemctl reboot)" >/dev/null 2>&1 &' 2>&1
|
||||
echo "Reboot command sent at $(date +%H:%M:%S). Host will drop now."
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] WAITING FOR SSH TO RETURN (max 10 minutes)"
|
||||
echo "================================================================"
|
||||
DEADLINE=$(( $(date +%s) + 600 ))
|
||||
LAST_PRINT=0
|
||||
while [ "$(date +%s)" -lt "$DEADLINE" ]; do
|
||||
now=$(date +%s)
|
||||
# Print a heartbeat every 15s
|
||||
if [ $((now - LAST_PRINT)) -ge 15 ]; then
|
||||
elapsed=$((DEADLINE - now - 600)); elapsed=${elapsed#-}
|
||||
echo " [$(date +%H:%M:%S)] still waiting... (${elapsed}s elapsed)"
|
||||
LAST_PRINT=$now
|
||||
fi
|
||||
# Try SSH
|
||||
if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " [$(date +%H:%M:%S)] SSH is back!"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Final check
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " [$(date +%H:%M:%S)] FAILED: host not reachable after 10 minutes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Give services a moment to settle after SSH returns
|
||||
echo " Waiting 20s for services to settle..."
|
||||
sleep 20
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] POST-REBOOT VERIFICATION"
|
||||
echo "================================================================"
|
||||
echo "--- Uptime ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime' 2>&1
|
||||
echo ""
|
||||
echo "--- TCP congestion control ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc' 2>&1
|
||||
echo ""
|
||||
echo "--- vm.swappiness ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl vm.swappiness' 2>&1
|
||||
echo ""
|
||||
echo "--- scaling_governor ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "(no cpufreq driver)"' 2>&1
|
||||
echo ""
|
||||
echo "--- NFS mount options (looking for nconnect + noatime) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -20' 2>&1
|
||||
echo ""
|
||||
echo "--- NFS TCP connection count (expect ~8 = 4 per server with nconnect=4) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l' 2>&1
|
||||
echo ""
|
||||
echo "--- Running VMs ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null' 2>&1
|
||||
echo ""
|
||||
echo "--- Failed services? ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'systemctl --failed --no-legend 2>/dev/null | head -10' 2>&1
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] DONE"
|
||||
echo "================================================================"
|
||||
Executable
+141
@@ -0,0 +1,141 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# apply-bond-hash.sh
|
||||
#
|
||||
# Adds bond-xmit-hash-policy layer3+4 to bond0 in /etc/network/interfaces,
|
||||
# then reloads networking with ifreload -a.
|
||||
#
|
||||
# SSH survivability: this is safe IF your SSH session is on vmbr0/nic0
|
||||
# (management network), NOT on bond0/datanet (storage network).
|
||||
# tsys7's topology confirms this: SSH comes in on vmbr0 (nic0).
|
||||
#
|
||||
# Safety:
|
||||
# - Dry-run by default (--apply to commit)
|
||||
# - Full backup of /etc/network/interfaces
|
||||
# - Generates rollback script
|
||||
# - Does NOT reboot — uses ifreload -a which is hot-reload
|
||||
###############################################################################
|
||||
set -euo pipefail
|
||||
|
||||
HOST="$(hostname -s)"
|
||||
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
||||
ROLLBACK="/root/perfopt-bond-rollback-${TS_SHORT}.sh"
|
||||
ACTION="${1:-dryrun}"
|
||||
|
||||
[ "$ACTION" = "--apply" ] && ACTION="apply" || ACTION="dryrun"
|
||||
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " apply-bond-hash — $HOST"
|
||||
echo " mode: $ACTION"
|
||||
echo "==================================================================="
|
||||
|
||||
if [ ! -r /etc/network/interfaces ]; then
|
||||
echo "FATAL: /etc/network/interfaces not readable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if bond0 exists in the config
|
||||
if ! grep -q 'bond0' /etc/network/interfaces; then
|
||||
echo "No bond0 found in /etc/network/interfaces — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Check if the hash policy is already set
|
||||
if grep -q 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces; then
|
||||
echo "bond-xmit-hash-policy already present:"
|
||||
grep 'bond-xmit-hash-policy\|xmit_hash_policy' /etc/network/interfaces
|
||||
echo "Checking value..."
|
||||
if grep -q 'layer3+4' /etc/network/interfaces; then
|
||||
echo "Already set to layer3+4 — nothing to do."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
# Show current bond state
|
||||
echo ""
|
||||
echo "--- Current bond0 state ---"
|
||||
cat /proc/net/bonding/bond0 2>/dev/null | head -15 || echo "(bond0 not up or not present)"
|
||||
|
||||
# Back up
|
||||
cp -a /etc/network/interfaces "$BACKUP_DIR/interfaces"
|
||||
|
||||
# Generate rollback script
|
||||
cat > "$ROLLBACK" <<EOF
|
||||
#!/bin/bash
|
||||
# Rollback for bond-xmit-hash-policy change
|
||||
# Restores original /etc/network/interfaces and reloads
|
||||
set -euo pipefail
|
||||
cp -a "$BACKUP_DIR/interfaces" /etc/network/interfaces
|
||||
echo "Restored /etc/network/interfaces"
|
||||
echo "Reloading networking..."
|
||||
ifreload -a 2>&1 || systemctl restart networking 2>&1 || true
|
||||
echo "Done. bond0 hash policy reverted to original."
|
||||
EOF
|
||||
chmod +x "$ROLLBACK"
|
||||
|
||||
echo ""
|
||||
echo "--- Proposed change ---"
|
||||
echo "Add line ' bond-xmit-hash-policy layer3+4' to the bond0 stanza."
|
||||
echo ""
|
||||
|
||||
if [ "$ACTION" != "apply" ]; then
|
||||
echo "DRY RUN — no changes made."
|
||||
echo "To commit: bash $0 --apply"
|
||||
echo "Rollback script (pre-generated): $ROLLBACK"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Apply: use sed to insert bond-xmit-hash-policy after bond-mode line
|
||||
# The bond0 stanza looks like:
|
||||
# auto bond0
|
||||
# iface bond0 inet manual
|
||||
# bond-slaves nic1 nic2
|
||||
# bond-miimon 100
|
||||
# bond-mode 802.3ad
|
||||
#
|
||||
# We insert after the bond-mode line.
|
||||
echo "Applying..."
|
||||
|
||||
# Check if bond-mode line exists (various formats)
|
||||
if grep -qE '^\s*bond-mode\s+802.3ad' /etc/network/interfaces; then
|
||||
# Insert after bond-mode 802.3ad line
|
||||
sed -i '/^\s*bond-mode\s+802\.3ad/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode line."
|
||||
elif grep -qE '^\s*bond-mode\s+4' /etc/network/interfaces; then
|
||||
sed -i '/^\s*bond-mode\s+4/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4 after bond-mode 4 line."
|
||||
else
|
||||
echo "Could not find bond-mode line — inserting after bond-slaves line instead."
|
||||
sed -i '/^\s*bond-slaves/a\\tbond-xmit-hash-policy layer3+4' /etc/network/interfaces
|
||||
fi
|
||||
|
||||
# Show the result
|
||||
echo ""
|
||||
echo "--- Updated bond0 stanza ---"
|
||||
awk '/^auto bond0/,/^$/' /etc/network/interfaces
|
||||
|
||||
echo ""
|
||||
echo "--- Reloading networking (ifreload -a) ---"
|
||||
echo "SSH should survive (it's on vmbr0/nic0, not bond0)..."
|
||||
ifreload -a 2>&1 || {
|
||||
echo "ifreload failed, trying systemctl restart networking..."
|
||||
systemctl restart networking 2>&1
|
||||
}
|
||||
|
||||
# Wait a moment for bond to renegotiate
|
||||
echo "Waiting 5s for LACP to renegotiate..."
|
||||
sleep 5
|
||||
|
||||
echo ""
|
||||
echo "--- Post-change bond0 state ---"
|
||||
cat /proc/net/bonding/bond0 2>/dev/null | head -20
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " DONE."
|
||||
echo " Backup: $BACKUP_DIR/interfaces"
|
||||
echo " Rollback: bash $ROLLBACK"
|
||||
echo "==================================================================="
|
||||
Executable
+439
@@ -0,0 +1,439 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# apply-tunings.sh
|
||||
#
|
||||
# Applies the Tier 0 host-side tunings identified in FINDINGS.md:
|
||||
# 1. Sets scaling_governor=performance on every CPU
|
||||
# 2. Sets vm.swappiness appropriately (1 on storage hosts, 10 on VM hosts)
|
||||
# 3. Enables tcp_bbr module + sets congestion control
|
||||
# 4. Sets net.core.rmem_max/wmem_max + tcp_rmem/wmem for high-BDP NFS
|
||||
# 5. Sets the recommended tuned-adm profile (network-throughput or virtual-host)
|
||||
# 6. Adds nconnect=4 + noatime to NFS client mounts (edits /etc/pve/storage.cfg)
|
||||
# 7. (does NOT touch bond0 xmit-hash — that requires ifreload/network
|
||||
# restart which drops the host. Provided as a separate --emit-bond-patch
|
||||
# flag that PRINTS the change but does not apply it.)
|
||||
#
|
||||
# Safety features:
|
||||
# - Dry-run mode by default (--apply to commit)
|
||||
# - Full backup of every modified file to /root/perfopt-backup-<timestamp>/
|
||||
# - Per-host behaviour: detects storage hosts by hostname and applies the
|
||||
# right profile (tsys4, tsys5 = storage; others = VM hosts).
|
||||
# - Generates a /root/perfopt-rollback.sh that undoes everything.
|
||||
# - Does NOT reboot, does NOT restart networking, does NOT touch hardware.
|
||||
#
|
||||
# Usage:
|
||||
# bash apply-tunings.sh # dry run, show what would change
|
||||
# bash apply-tunings.sh --apply # commit changes
|
||||
# bash apply-tunings.sh --rollback # restore from latest backup
|
||||
# bash apply-tunings.sh --emit-bond-patch # show bond0 hash change (no apply)
|
||||
###############################################################################
|
||||
set -u
|
||||
umask 022
|
||||
|
||||
HOST="$(hostname -s)"
|
||||
TS="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
TS_SHORT="$(date +%Y%m%d-%H%M%S)"
|
||||
BACKUP_DIR="/root/perfopt-backup-${TS_SHORT}"
|
||||
ROLLBACK_SCRIPT="/root/perfopt-rollback-${TS_SHORT}.sh"
|
||||
ACTION="dryrun"
|
||||
SKIP_NFS=0
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--apply) ACTION="apply" ;;
|
||||
--rollback) ACTION="rollback" ;;
|
||||
--emit-bond-patch) ACTION="bond-patch" ;;
|
||||
--no-nfs) SKIP_NFS=1 ;;
|
||||
-h|--help) sed -n '2,35p' "$0"; exit 0 ;;
|
||||
*) echo "unknown arg: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Detect host role from hostname
|
||||
case "$HOST" in
|
||||
pfv-tsys4|pfv-tsys5)
|
||||
HOST_ROLE="storage"
|
||||
TUNED_PROFILE="network-throughput"
|
||||
SWAPPINESS="1"
|
||||
;;
|
||||
pfv-tsys1|pfv-tsys3|pfv-tsys6|pfv-tsys7)
|
||||
HOST_ROLE="vmhost"
|
||||
TUNED_PROFILE="virtual-host"
|
||||
SWAPPINESS="10"
|
||||
;;
|
||||
*)
|
||||
HOST_ROLE="unknown"
|
||||
TUNED_PROFILE="virtual-host"
|
||||
SWAPPINESS="10"
|
||||
;;
|
||||
esac
|
||||
|
||||
# --- rollback path ---------------------------------------------------------
|
||||
if [ "$ACTION" = "rollback" ]; then
|
||||
echo "Looking for latest backup under /root/perfopt-backup-* ..."
|
||||
latest=""
|
||||
while IFS= read -r d; do
|
||||
latest="$d"
|
||||
done < <(find /root -maxdepth 1 -type d -name 'perfopt-backup-*' 2>/dev/null | sort | tail -n1)
|
||||
if [ -z "$latest" ]; then
|
||||
echo "No backup found under /root/perfopt-backup-*" >&2
|
||||
exit 1
|
||||
fi
|
||||
rb="$latest/perfopt-rollback.sh"
|
||||
if [ ! -x "$rb" ] && [ ! -r "$rb" ]; then
|
||||
echo "Rollback script missing in $latest" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "Rolling back using: $rb"
|
||||
bash "$rb"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
# --- bond patch print-only path -------------------------------------------
|
||||
if [ "$ACTION" = "bond-patch" ]; then
|
||||
echo "==================================================================="
|
||||
echo " Proposed bond0 xmit_hash_policy change"
|
||||
echo "==================================================================="
|
||||
echo
|
||||
if [ ! -r /etc/network/interfaces ]; then
|
||||
echo "/etc/network/interfaces not readable"
|
||||
exit 1
|
||||
fi
|
||||
if ! grep -q 'bond-mode 802.3ad\|bond-mode 4\|bond-slaves' /etc/network/interfaces; then
|
||||
echo "No bond0 detected on this host — nothing to patch."
|
||||
exit 0
|
||||
fi
|
||||
cat <<EOF
|
||||
A manual edit to /etc/network/interfaces is required. The bond0 stanza
|
||||
needs this line added (it defaults to layer2, which is wrong for storage):
|
||||
|
||||
bond-xmit-hash-policy layer3+4
|
||||
|
||||
After editing, you MUST reload networking for it to take effect:
|
||||
ifreload -a # safe, brings interfaces down/up
|
||||
# OR
|
||||
systemctl restart networking # heavier, briefly drops connections
|
||||
|
||||
WARNING: applying this on a remote host over bond0 will briefly drop your
|
||||
SSH session. Run from console/IPMI, or schedule a maintenance window.
|
||||
|
||||
The change is reversible by removing the line and reloading again.
|
||||
EOF
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --- main path (dryrun or apply) ------------------------------------------
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Emit the rollback script header
|
||||
cat > "$ROLLBACK_SCRIPT" <<EOF
|
||||
#!/bin/bash
|
||||
# Auto-generated rollback for perfopt apply-tunings.sh
|
||||
# Backup timestamp: $TS
|
||||
# Backup dir: $BACKUP_DIR
|
||||
# Generated on: $HOST
|
||||
set -u
|
||||
EOF
|
||||
chmod +x "$ROLLBACK_SCRIPT"
|
||||
|
||||
backup_file() {
|
||||
local f="$1"
|
||||
if [ -e "$f" ]; then
|
||||
cp -a "$f" "$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
||||
fi
|
||||
}
|
||||
|
||||
# Helper: append a restore command to the rollback script
|
||||
rb_restore() {
|
||||
local f="$1"
|
||||
local bk
|
||||
bk="$BACKUP_DIR/$(echo "$f" | sed 's|^/||; s|/|__|g')"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
if [ -r "$bk" ]; then
|
||||
cp -a "$bk" "$f" && echo "restored $f"
|
||||
else
|
||||
echo "WARN: backup $bk missing — $f left as-is"
|
||||
fi
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: append a sysctl restore command
|
||||
rb_sysctl_restore() {
|
||||
local key="$1" orig_val="$2"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
sysctl -w "$key=$orig_val" >/dev/null && echo "restored $key=$orig_val"
|
||||
EOF
|
||||
}
|
||||
|
||||
# Helper: append a tuned-adm restore
|
||||
rb_tuned_restore() {
|
||||
local profile="$1"
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
tuned-adm profile "$profile" 2>/dev/null && echo "restored tuned profile: $profile"
|
||||
EOF
|
||||
}
|
||||
|
||||
echo "==================================================================="
|
||||
echo " perfopt apply-tunings — $HOST"
|
||||
echo " mode: $ACTION"
|
||||
echo " role: $HOST_ROLE (tuned=$TUNED_PROFILE, swappiness=$SWAPPINESS)"
|
||||
echo " backup: $BACKUP_DIR"
|
||||
echo " rollback: $ROLLBACK_SCRIPT"
|
||||
echo "==================================================================="
|
||||
echo
|
||||
|
||||
# ===========================================================================
|
||||
# 1. scaling_governor → performance
|
||||
# ===========================================================================
|
||||
echo "--- 1. CPU scaling_governor → performance"
|
||||
if [ -r /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor ]; then
|
||||
cur=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor)
|
||||
echo " current: $cur"
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
if [ "$cur" != "performance" ]; then
|
||||
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||
echo performance > "$c" 2>/dev/null || true
|
||||
done
|
||||
# Persist via systemd tmpfiles / sysctl fallback
|
||||
cat > /etc/systemd/system/perfopt-cpu-performance.service <<EOF
|
||||
[Unit]
|
||||
Description=Set CPU scaling_governor=performance (perfopt)
|
||||
After=multi-user.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/sh -c 'for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo performance > "\$c" 2>/dev/null || true; done'
|
||||
RemainAfterExit=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
systemctl daemon-reload
|
||||
systemctl enable perfopt-cpu-performance.service 2>/dev/null
|
||||
systemctl start perfopt-cpu-performance.service 2>/dev/null
|
||||
echo " applied + persisted via systemd unit"
|
||||
# Rollback
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
for c in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do echo "$cur" > "\$c" 2>/dev/null || true; done
|
||||
systemctl disable --now perfopt-cpu-performance.service 2>/dev/null || true
|
||||
rm -f /etc/systemd/system/perfopt-cpu-performance.service
|
||||
systemctl daemon-reload
|
||||
echo "restored scaling_governor=$cur (best-effort; original may have been dynamic)"
|
||||
EOF
|
||||
else
|
||||
echo " already performance — no change"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " cpufreq driver not loaded — nothing to do (typical on BIOS-locked servers)"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 2. vm.swappiness
|
||||
# ===========================================================================
|
||||
echo "--- 2. vm.swappiness → $SWAPPINESS (role=$HOST_ROLE)"
|
||||
cur_swappiness=$(sysctl -n vm.swappiness 2>/dev/null || echo "?")
|
||||
echo " current: $cur_swappiness"
|
||||
if [ "$ACTION" = "apply" ] && [ "$cur_swappiness" != "$SWAPPINESS" ]; then
|
||||
# Persist via sysctl.d
|
||||
backup_file /etc/sysctl.d/99-perfopt.conf
|
||||
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
||||
vm.swappiness = $SWAPPINESS
|
||||
EOF
|
||||
sysctl -w "vm.swappiness=$SWAPPINESS" >/dev/null
|
||||
echo " applied (persisted to /etc/sysctl.d/99-perfopt.conf)"
|
||||
rb_restore /etc/sysctl.d/99-perfopt.conf
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
rm -f /etc/sysctl.d/99-perfopt.conf
|
||||
sysctl -w "vm.swappiness=$cur_swappiness" >/dev/null
|
||||
echo "restored vm.swappiness=$cur_swappiness"
|
||||
EOF
|
||||
else
|
||||
echo " no change"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 3. TCP BBR + buffers (single sysctl.d file)
|
||||
# ===========================================================================
|
||||
echo "--- 3. TCP BBR + socket buffers"
|
||||
cur_cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null || echo "?")
|
||||
cur_avail=$(sysctl -n net.ipv4.tcp_available_congestion_control 2>/dev/null || echo "?")
|
||||
echo " current: $cur_cc (available: $cur_avail)"
|
||||
|
||||
# Decide sysctl values (these are the consensus values for a 1-10 GbE NFS host)
|
||||
TARGET_RMEM_MAX=134217728 # 128 MB
|
||||
TARGET_WMEM_MAX=134217728
|
||||
TARGET_RMEM_DEFAULT=26214400 # 25 MB
|
||||
TARGET_WMEM_DEFAULT=26214400
|
||||
TARGET_TCP_RMEM="4096 87380 $TARGET_RMEM_MAX"
|
||||
TARGET_TCP_WMEM="4096 65536 $TARGET_RMEM_MAX"
|
||||
TARGET_NETDEV_MAX_BACKLOG=250000
|
||||
TARGET_SOMAXCONN=65535
|
||||
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
# Ensure tcp_bbr module loads at boot
|
||||
if ! grep -q '^tcp_bbr' /etc/modules-load.d/modules.conf 2>/dev/null; then
|
||||
backup_file /etc/modules-load.d/modules.conf
|
||||
mkdir -p /etc/modules-load.d
|
||||
cat >> /etc/modules-load.d/modules.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS
|
||||
tcp_bbr
|
||||
EOF
|
||||
modprobe tcp_bbr 2>/dev/null || true
|
||||
rb_restore /etc/modules-load.d/modules.conf
|
||||
fi
|
||||
|
||||
backup_file /etc/sysctl.d/99-perfopt.conf
|
||||
cat > /etc/sysctl.d/99-perfopt.conf <<EOF
|
||||
# perfopt apply-tunings.sh — $TS — host=$HOST role=$HOST_ROLE
|
||||
vm.swappiness = $SWAPPINESS
|
||||
|
||||
# TCP BBR + high-BDP buffers (good for NFS over 1-10 GbE)
|
||||
net.ipv4.tcp_congestion_control = bbr
|
||||
net.core.default_qdisc = fq
|
||||
net.core.rmem_max = $TARGET_RMEM_MAX
|
||||
net.core.wmem_max = $TARGET_WMEM_MAX
|
||||
net.core.rmem_default = $TARGET_RMEM_DEFAULT
|
||||
net.core.wmem_default = $TARGET_WMEM_DEFAULT
|
||||
net.core.netdev_max_backlog = $TARGET_NETDEV_MAX_BACKLOG
|
||||
net.core.somaxconn = $TARGET_SOMAXCONN
|
||||
net.ipv4.tcp_rmem = $TARGET_TCP_RMEM
|
||||
net.ipv4.tcp_wmem = $TARGET_TCP_WMEM
|
||||
EOF
|
||||
sysctl --system >/dev/null 2>&1
|
||||
echo " applied (written to /etc/sysctl.d/99-perfopt.conf + sysctl --system)"
|
||||
# Rebuild rollback for sysctl
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
# Restore original sysctl state
|
||||
rm -f /etc/sysctl.d/99-perfopt.conf
|
||||
EOF
|
||||
rb_sysctl_restore "net.ipv4.tcp_congestion_control" "$cur_cc"
|
||||
rb_sysctl_restore "vm.swappiness" "$cur_swappiness"
|
||||
# Note: original buffer sizes not all captured; restoration is best-effort.
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
echo "Note: net buffers restored to kernel defaults (original values not captured)."
|
||||
echo "Run 'sysctl --system' to apply."
|
||||
sysctl --system >/dev/null 2>&1 || true
|
||||
EOF
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 4. tuned-adm profile
|
||||
# ===========================================================================
|
||||
echo "--- 4. tuned-adm profile → $TUNED_PROFILE"
|
||||
if command -v tuned-adm >/dev/null 2>&1; then
|
||||
cur_profile=$(tuned-adm active 2>/dev/null | awk -F: '/Current active profile/{gsub(/^[ \t]+/,"",$2); print $2}')
|
||||
echo " current: $cur_profile"
|
||||
if [ "$ACTION" = "apply" ] && [ "$cur_profile" != "$TUNED_PROFILE" ]; then
|
||||
tuned-adm profile "$TUNED_PROFILE" 2>&1 | sed 's/^/ /'
|
||||
rb_tuned_restore "$cur_profile"
|
||||
else
|
||||
echo " no change"
|
||||
fi
|
||||
else
|
||||
echo " tuned-adm not installed — skipping"
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 5. NFS mount option hardening
|
||||
# Edit /etc/pve/storage.cfg to add nconnect=4 + noatime to NFS plugins.
|
||||
# Proxmox applies these on next mount/remount.
|
||||
# Skipped with --no-nfs (for NFS servers where we don't want to remount).
|
||||
# ===========================================================================
|
||||
echo "--- 5. NFS mount options: add nconnect=4, noatime"
|
||||
if [ "$SKIP_NFS" -eq 1 ]; then
|
||||
echo " SKIPPED (--no-nfs specified)"
|
||||
echo " Note: storage.cfg is cluster-wide; if edited on another host,"
|
||||
echo " the options are already staged here and activate on next reboot."
|
||||
else
|
||||
if [ -r /etc/pve/storage.cfg ]; then
|
||||
backup_file /etc/pve/storage.cfg
|
||||
if grep -q '^nfs:' /etc/pve/storage.cfg; then
|
||||
# Count how many nfs: stanzas lack the options
|
||||
while IFS= read -r line; do
|
||||
if echo "$line" | grep -q '^nfs:'; then
|
||||
cur_stanza=$(echo "$line" | awk '{print $2}')
|
||||
# Look ahead for the next few lines to see if options already set
|
||||
echo " nfs stanza: $cur_stanza"
|
||||
fi
|
||||
done < /etc/pve/storage.cfg
|
||||
|
||||
# Check whether any nfs stanza already has options
|
||||
if grep -A1 '^nfs:' /etc/pve/storage.cfg | grep -q 'options.*nconnect=4'; then
|
||||
echo " some stanzas already have nconnect=4 — check manually"
|
||||
else
|
||||
echo " proposed change: add 'options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2' to each nfs stanza"
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
# Apply via perl for safety (in-place edit with backup)
|
||||
if cp -a /etc/pve/storage.cfg "$BACKUP_DIR/etc__pve__storage.cfg"; then
|
||||
# Use a python helper for robust PVE storage.cfg editing
|
||||
python3 - <<PYEOF
|
||||
import re, pathlib
|
||||
p = pathlib.Path("/etc/pve/storage.cfg")
|
||||
text = p.read_text()
|
||||
pattern = re.compile(r'(^nfs:\s*\S+\n(?:[ \t]+[^\n]+\n)+)', re.MULTILINE)
|
||||
modified = 0
|
||||
def fix(m):
|
||||
global modified
|
||||
block = m.group(1)
|
||||
if 'options' in block:
|
||||
return block
|
||||
lines = block.rstrip('\n').split('\n')
|
||||
lines.insert(1, '\toptions nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2')
|
||||
modified += 1
|
||||
return '\n'.join(lines) + '\n'
|
||||
new_text = pattern.sub(fix, text)
|
||||
if modified:
|
||||
p.write_text(new_text)
|
||||
print(f" patched {modified} NFS stanzas with options line")
|
||||
else:
|
||||
print(" no NFS stanza needed patching (all already had options or no nfs: found)")
|
||||
PYEOF
|
||||
# Remount existing NFS mounts to pick up new options
|
||||
echo " remounting NFS mounts to apply new options..."
|
||||
while IFS= read -r mp; do
|
||||
mount -o remount "$mp" 2>/dev/null && echo " remounted $mp" || echo " FAILED to remount $mp (will pick up on next mount)"
|
||||
done < <(awk '$3 ~ /^nfs/{print $2}' /proc/mounts 2>/dev/null | sort -u)
|
||||
rb_restore /etc/pve/storage.cfg
|
||||
cat >> "$ROLLBACK_SCRIPT" <<EOF
|
||||
# To fully undo NFS options, also remount:
|
||||
while IFS= read -r mp; do
|
||||
mount -o remount "\$mp" 2>/dev/null
|
||||
done < <(awk '\$3 ~ /^nfs/{print \$2}' /proc/mounts 2>/dev/null | sort -u)
|
||||
echo "restored NFS mount options"
|
||||
EOF
|
||||
else
|
||||
echo " ERROR: could not back up storage.cfg — aborting NFS patch"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
else
|
||||
echo " no NFS stanzas in storage.cfg — skipping"
|
||||
fi
|
||||
else
|
||||
echo " /etc/pve/storage.cfg not readable — skipping"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ===========================================================================
|
||||
# 6. Summary
|
||||
# ===========================================================================
|
||||
echo
|
||||
echo "==================================================================="
|
||||
if [ "$ACTION" = "apply" ]; then
|
||||
echo " DONE. Backups in: $BACKUP_DIR"
|
||||
echo " Rollback: bash $ROLLBACK_SCRIPT"
|
||||
echo
|
||||
echo " Next steps:"
|
||||
echo " - Verify with 'sysctl net.ipv4.tcp_congestion_control vm.swappiness'"
|
||||
echo " - Verify NFS mounts with 'nfsstat -m' (look for nconnect=4)"
|
||||
echo " - Run 'bash apply-tunings.sh --emit-bond-patch' for the bond0"
|
||||
echo " xmit_hash_policy change (requires network restart, do in"
|
||||
echo " maintenance window)"
|
||||
else
|
||||
echo " DRY RUN — no changes made."
|
||||
echo " To commit: bash apply-tunings.sh --apply"
|
||||
fi
|
||||
echo "==================================================================="
|
||||
Executable
+1138
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,162 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
conman-console.py — Drive a serial console via a conman server, read-only.
|
||||
|
||||
Connects to a conmand server (RFC 127-style multiplexer) over the network,
|
||||
opens a named console, sends the commands from a .cmds file, captures all
|
||||
output, and disconnects. Requires no expect/tcl — pure stdlib PTY.
|
||||
|
||||
This replaces the old sw-capture-remote.sh workflow that killed the serial
|
||||
device holder (conflicting with conman/ser2net). Instead, it talks to conman
|
||||
over TCP, which multiplexes safely with other sessions.
|
||||
|
||||
All endpoints are configurable via environment variables so this works on
|
||||
any network with a conman server:
|
||||
|
||||
CONMAN_SERVER conman server host:port (default: via CONSOLE_HOST)
|
||||
CONSOLE console name to open (required)
|
||||
CMDS_FILE file of commands to send (required)
|
||||
TIMEOUT overall timeout in seconds (default: 45)
|
||||
CMD_DELAY seconds between commands (default: 3)
|
||||
WAKE_DELAY seconds after connect (default: 2)
|
||||
|
||||
Usage:
|
||||
CONMAN_SERVER=console-host:7890 \\
|
||||
python3 conman-console.py --console pfv-core-sw01 --cmds switches/pfv-core-sw01.cmds
|
||||
|
||||
Lines starting with '!' or '#' in the cmds file are comments (skipped).
|
||||
Blank lines are skipped. The conman escape sequence (&.) is sent automatically
|
||||
to disconnect. A password prompt aborts immediately (we never send creds).
|
||||
|
||||
Exit codes:
|
||||
0 clean run
|
||||
1 usage / setup error
|
||||
2 could not connect to conman server
|
||||
3 timeout (partial output still printed)
|
||||
4 password prompt encountered (aborted)
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import pty
|
||||
import re
|
||||
import select
|
||||
import sys
|
||||
import time
|
||||
|
||||
PWD_RE = re.compile(rb"[Pp]assword:\s*$")
|
||||
MORE_RE = re.compile(rb"--\s*[Mm]ore\s*--|[Mm]ore:\s*<space>")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description="Drive a conman console session read-only via PTY")
|
||||
ap.add_argument("--console", required=True,
|
||||
help="console name (e.g. pfv-core-sw01)")
|
||||
ap.add_argument("--cmds", required=True,
|
||||
help="command file (one command per line; !/# = comment)")
|
||||
ap.add_argument("--server",
|
||||
default=os.environ.get("CONMAN_SERVER", ""),
|
||||
help="conman server host:port (env: CONMAN_SERVER)")
|
||||
ap.add_argument("--timeout", type=int,
|
||||
default=int(os.environ.get("TIMEOUT", "45")),
|
||||
help="overall timeout seconds (env: TIMEOUT)")
|
||||
ap.add_argument("--cmd-delay", type=float,
|
||||
default=float(os.environ.get("CMD_DELAY", "3")),
|
||||
help="seconds between commands (env: CMD_DELAY)")
|
||||
ap.add_argument("--wake-delay", type=float,
|
||||
default=float(os.environ.get("WAKE_DELAY", "2")),
|
||||
help="seconds after connect before first command (env: WAKE_DELAY)")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.server:
|
||||
sys.stderr.write("ERROR: --server or CONMAN_SERVER env required\n")
|
||||
return 1
|
||||
|
||||
with open(args.cmds) as f:
|
||||
cmds = [l.strip() for l in f
|
||||
if l.strip() and not l.strip().startswith(("!", "#"))]
|
||||
|
||||
pid, fd = pty.fork()
|
||||
if pid == 0:
|
||||
try:
|
||||
os.execvp("conman",
|
||||
["conman", "-d", args.server, "-f", args.console])
|
||||
except OSError as e:
|
||||
sys.stderr.write(f"ERROR: cannot exec conman: {e}\n")
|
||||
os._exit(2)
|
||||
os._exit(2)
|
||||
|
||||
output = b""
|
||||
cmd_queue = list(cmds)
|
||||
sent_disconnect = False
|
||||
start = time.time()
|
||||
last_action = 0.0
|
||||
phase = "connect"
|
||||
|
||||
while time.time() - start < args.timeout:
|
||||
ready, _, _ = select.select([fd], [], [], 0.5)
|
||||
if ready:
|
||||
try:
|
||||
data = os.read(fd, 8192)
|
||||
except OSError:
|
||||
break
|
||||
if not data:
|
||||
break
|
||||
output += data
|
||||
|
||||
if PWD_RE.search(output.split(b"\n")[-1] if output else b""):
|
||||
sys.stderr.write("[ABORT] password prompt detected — "
|
||||
"never sending credentials\n")
|
||||
os.write(fd, b"&.\n")
|
||||
break
|
||||
|
||||
# Handle pagination: send space to continue
|
||||
if MORE_RE.search(output[-200:] if output else b""):
|
||||
os.write(fd, b" ")
|
||||
time.sleep(0.5)
|
||||
|
||||
elapsed = time.time() - start
|
||||
gap = elapsed - last_action
|
||||
|
||||
if phase == "connect" and gap >= args.wake_delay:
|
||||
os.write(fd, b"\n")
|
||||
phase = "send"
|
||||
last_action = elapsed
|
||||
elif phase == "send" and gap >= args.cmd_delay:
|
||||
if cmd_queue:
|
||||
cmd = cmd_queue.pop(0)
|
||||
os.write(fd, (cmd + "\n").encode())
|
||||
last_action = elapsed
|
||||
else:
|
||||
phase = "drain"
|
||||
last_action = elapsed
|
||||
elif phase == "drain" and gap >= args.cmd_delay:
|
||||
os.write(fd, b"&.\n")
|
||||
sent_disconnect = True
|
||||
phase = "done"
|
||||
last_action = elapsed
|
||||
elif phase == "done" and gap >= 2:
|
||||
break
|
||||
|
||||
if not sent_disconnect:
|
||||
try:
|
||||
os.write(fd, b"&.\n")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.waitpid(pid, 0)
|
||||
except ChildProcessError:
|
||||
pass
|
||||
|
||||
sys.stdout.buffer.write(output)
|
||||
sys.stdout.flush()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+86
@@ -0,0 +1,86 @@
|
||||
#!/bin/bash
|
||||
# fix-tsys7.sh - fixes the two issues found: NFS options string + bond hash
|
||||
# Runs on the target host directly.
|
||||
set -euo pipefail
|
||||
|
||||
echo "==================================================================="
|
||||
echo " FIX: NFS options + bond hash policy"
|
||||
echo "==================================================================="
|
||||
|
||||
# --- FIX 1: Simplify NFS options in storage.cfg ---------------------------
|
||||
echo ""
|
||||
echo "--- FIX 1: Simplify NFS options (remove version=4.2 conflict) ---"
|
||||
# Replace the overly-complex options line with a minimal one
|
||||
# PVE handles vers/rsize/wsize/hard/etc internally; we only need nconnect + noatime
|
||||
if grep -q 'options.*nconnect=4.*version=4.2' /etc/pve/storage.cfg; then
|
||||
# Use sed to replace each options line
|
||||
sed -i 's/options nconnect=4,noatime,rsize=1048576,wsize=1048576,hard,timeo=600,retrans=2,version=4.2/options nconnect=4,noatime/' /etc/pve/storage.cfg
|
||||
echo "Fixed: simplified NFS options to 'nconnect=4,noatime'"
|
||||
else
|
||||
echo "Options line already simplified or not present"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Verify storage.cfg NFS stanzas ---"
|
||||
grep -A 2 "^nfs: D2" /etc/pve/storage.cfg | head -3
|
||||
echo "..."
|
||||
grep -c "options nconnect" /etc/pve/storage.cfg
|
||||
echo " NFS stanzas with options"
|
||||
|
||||
# --- FIX 2: Bond hash policy (force via sysfs + persist in interfaces) ----
|
||||
echo ""
|
||||
echo "--- FIX 2: Apply bond xmit_hash_policy=layer3+4 ---"
|
||||
|
||||
# 2a: Apply LIVE via sysfs (takes effect immediately, no network reload)
|
||||
echo "Applying live via sysfs..."
|
||||
if echo "layer3+4" > /sys/class/net/bond0/bonding/xmit_hash_policy 2>/dev/null; then
|
||||
echo "Live sysfs apply: SUCCESS"
|
||||
else
|
||||
echo "Live sysfs apply: FAILED (will persist in config and apply on ifreload)"
|
||||
fi
|
||||
|
||||
# Verify live state
|
||||
echo ""
|
||||
echo "Live bond0 hash policy:"
|
||||
cat /proc/net/bonding/bond0 | grep "Transmit Hash"
|
||||
|
||||
# 2b: Persist in /etc/network/interfaces (fix the sed that failed before)
|
||||
echo ""
|
||||
echo "Persisting in /etc/network/interfaces..."
|
||||
|
||||
# Backup
|
||||
cp -a /etc/network/interfaces "/root/interfaces.bondfix.$(date +%Y%m%d%H%M%S)"
|
||||
|
||||
# Check if already present
|
||||
if grep -q 'bond-xmit-hash-policy' /etc/network/interfaces; then
|
||||
echo "bond-xmit-hash-policy already in interfaces file"
|
||||
else
|
||||
# Use awk to insert after the bond-mode line (more reliable than sed)
|
||||
# Match any line containing 'bond-mode' (regardless of indentation)
|
||||
awk '
|
||||
/bond-mode/ && !done {
|
||||
print
|
||||
print "\tbond-xmit-hash-policy layer3+4"
|
||||
done=1
|
||||
next
|
||||
}
|
||||
{ print }
|
||||
' /etc/network/interfaces > /etc/network/interfaces.new
|
||||
|
||||
mv /etc/network/interfaces.new /etc/network/interfaces
|
||||
echo "Inserted bond-xmit-hash-policy layer3+4"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "--- Updated bond0 stanza ---"
|
||||
awk '/^auto bond0/,/^$/' /etc/network/interfaces
|
||||
|
||||
# Final verify
|
||||
echo ""
|
||||
echo "--- Final bond0 running state ---"
|
||||
cat /proc/net/bonding/bond0 | head -20
|
||||
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " FIX COMPLETE"
|
||||
echo "==================================================================="
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/bin/bash
|
||||
# identify-hosts.sh — identify all live hosts on a network via DNS/SNMP/SSH
|
||||
# Usage: bash identify-hosts.sh <hosts-file-or-subnet>
|
||||
# bash identify-hosts.sh 192.168.0.0/22
|
||||
# bash identify-hosts.sh /tmp/live-hosts.txt
|
||||
set -u
|
||||
COMMUNITY="${SNMP_COMMUNITY:-kn3lmgmt}"
|
||||
|
||||
if [ -f "${1:-}" ]; then
|
||||
HOSTS=$(cat "$1")
|
||||
elif [ -n "${1:-}" ]; then
|
||||
HOSTS=$(nmap -sn -n "$1" 2>/dev/null | grep 'Nmap scan report' | awk '{print $5}')
|
||||
else
|
||||
echo "Usage: $0 <hosts-file-or-subnet>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "%-16s %-30s %-25s %-20s\n" "IP" "DNS_NAME" "SNMP_SYSDESCR" "SSH_BANNER"
|
||||
printf "%-16s %-30s %-25s %-20s\n" "--" "-------" "-------------" "----------"
|
||||
|
||||
for ip in $HOSTS; do
|
||||
# DNS reverse
|
||||
rdns=$(dig +short -x "$ip" 2>/dev/null | head -1 | sed 's/\.$//')
|
||||
[ -z "$rdns" ] && rdns="(no-ptr)"
|
||||
|
||||
# SNMP sysDescr (first 25 chars)
|
||||
snmp=$(timeout 3 snmpget -Oqv -v2c -c "$COMMUNITY" "$ip" 1.3.6.1.2.1.1.1.0 2>/dev/null | head -1 | tr -d '"' | cut -c1-25)
|
||||
[ -z "$snmp" ] && snmp="-"
|
||||
|
||||
# SSH banner (quick)
|
||||
ssh_banner=$(timeout 3 bash -c "echo > /dev/tcp/$ip/22" 2>/dev/null && echo "ssh:open" || echo "-")
|
||||
|
||||
printf "%-16s %-30s %-25s %-20s\n" "$ip" "$rdns" "$snmp" "$ssh_banner"
|
||||
done
|
||||
@@ -0,0 +1,219 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# lacp-retrans-cause.sh (HOST-NATIVE)
|
||||
#
|
||||
# Runs on tsys6 (receiver) or tsys7 (sender). Auto-detects role.
|
||||
#
|
||||
# Question this answers: are the ~56K retransmits we see on 8-stream iperf
|
||||
# between tsys6 and tsys7 fixable (NIC ring drops, softnet drops, CPU
|
||||
# saturation) or just unavoidable LACP reordering overhead?
|
||||
#
|
||||
# Method: snapshot drop/error counters + softnet_stat + TCP SNMP before and
|
||||
# after a 12s iperf3 run, then print the deltas. The decisive columns are:
|
||||
# - NIC rx_dropped / rx_missed_errors / rx_no_dma_resources -> ring too small
|
||||
# - /proc/net/softnet_stat drops -> ksoftirq starved
|
||||
# - /proc/net/snmp TCP retranst -> TCP-level retrans
|
||||
# If NIC drop counters do NOT climb but TCP retrans does, the retrans are
|
||||
# coming from LACP reordering (out-of-order segments triggering fast
|
||||
# retransmit), not packet loss — and are NOT fixable by tuning.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
DURATION="${LACP_TEST_SECS:-12}"
|
||||
STREAMS="${LACP_STREAMS:-8}"
|
||||
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
||||
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
||||
LOG_DIR="/root"
|
||||
BOND="bond0"
|
||||
|
||||
HOST="$(uname -n)"; HOST="${HOST%%.*}"
|
||||
case "$HOST" in
|
||||
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-retrans-receiver.log" ;;
|
||||
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-retrans-sender.log" ;;
|
||||
*) echo "ERROR: not on tsys6 or tsys7 (host=$HOST)"; exit 2 ;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# ---- per-slave NIC counter snapshot -----------------------------------------
|
||||
# Captures rx_dropped, rx_missed_errors, rx_no_dma_resources, tx_retrans,
|
||||
# plus all error-like counters. Falls back gracefully if a counter doesn't
|
||||
# exist (different NIC drivers expose different names).
|
||||
nic_snapshot() {
|
||||
# shellcheck disable=SC2013 # interface names contain no spaces; word-splitting is safe here
|
||||
for s in $(awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND"); do
|
||||
[ -n "$s" ] || continue
|
||||
echo "[$s]"
|
||||
# ethtool -S may exist; show only drop/error/retrans lines
|
||||
if command -v ethtool >/dev/null 2>&1; then
|
||||
ethtool -S "$s" 2>/dev/null \
|
||||
| grep -iE 'drop|miss|error|no_dma|fifo|retrans|overflow' \
|
||||
|| echo "(ethtool -S: no matching counters or unsupported)"
|
||||
else
|
||||
echo "(ethtool not installed)"
|
||||
fi
|
||||
# /sys counters (always available, driver-agnostic)
|
||||
for c in rx_dropped tx_dropped rx_errors tx_errors rx_missed_errors \
|
||||
rx_length_errors rx_crc_errors rx_fifo_errors tx_fifo_errors \
|
||||
multicast collisions; do
|
||||
v=$(cat "/sys/class/net/$s/statistics/$c" 2>/dev/null)
|
||||
[ -n "$v" ] && printf ' sysfs %-22s = %s\n' "$c" "$v"
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# ---- softnet_stat snapshot (per-CPU RX softirq drops) -----------------------
|
||||
# /proc/net/softnet_stat columns are HEX. Col1=processed, col2=dropped,
|
||||
# col3=time_squeeze. We only care about dropped + squeeze (small numbers that
|
||||
# look identical in hex and decimal). Processed is informational only and we
|
||||
# don't try to delta it (hex arithmetic is awk-version-dependent).
|
||||
softnet_snapshot() {
|
||||
awk '{ printf "cpu%s processed=%s dropped=%s squeezed=%s\n", \
|
||||
NR-1, $1, $2, $3 }' /proc/net/softnet_stat
|
||||
}
|
||||
|
||||
# ---- TCP SNMP snapshot (the source of iperf's retransmit number) -----------
|
||||
tcp_snapshot() {
|
||||
awk '/^Tcp:/{
|
||||
if (!seen) {
|
||||
seen=1
|
||||
# /proc/net/snmp Tcp line 1 = names, line 2 = values
|
||||
n=split($0, names, " ")
|
||||
# re-find the values line
|
||||
getline
|
||||
vals=$0
|
||||
split(vals, v, " ")
|
||||
for (i=1;i<=n;i++) printf " %-22s = %s\n", names[i], v[i]
|
||||
}
|
||||
}' /proc/net/snmp
|
||||
}
|
||||
|
||||
# ---- print deltas for selected counters ------------------------------------
|
||||
# $1 = before file, $2 = after file
|
||||
# softnet_snapshot output: "cpuN processed=HEX dropped=HEX squeezed=HEX"
|
||||
# /proc/net/softnet_stat is hex, but dropped/squeezed are always small
|
||||
# integers (typically 0 on a healthy host), so +0 coercion is correct for
|
||||
# the values we care about. We deliberately DON'T report `processed` deltas
|
||||
# because hex arithmetic on large numbers is awk-version-dependent.
|
||||
softnet_delta() {
|
||||
awk '
|
||||
FNR==NR { if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
||||
split($0, p, /[= ]+/); drop[id]=p[5]+0; sqz[id]=p[7]+0 }
|
||||
next }
|
||||
{ if (match($0,/cpu[0-9]+/)) { id=substr($0,RSTART,RLENGTH);
|
||||
split($0, q, /[= ]+/);
|
||||
dd = (q[5]+0) - drop[id]
|
||||
sd = (q[7]+0) - sqz[id]
|
||||
if (dd != 0 || sd != 0)
|
||||
printf " %-8s dropped_delta=%-6s squeezed_delta=%-6s\n", id, dd, sd } }
|
||||
' "$1" "$2" | sort
|
||||
}
|
||||
|
||||
# $1 = before file, $2 = after file, $3 = section label prefix
|
||||
# File format: slave header line "[nic1]", then " counter_name = value" lines.
|
||||
# Output only counters whose value changed.
|
||||
nic_delta() {
|
||||
awk '
|
||||
FNR==NR { if ($0 ~ /^\[/) slave=$0
|
||||
else if ($0 ~ /=/) {
|
||||
# Everything before " = " is the counter name (trim spaces)
|
||||
pos = index($0, "=")
|
||||
name = substr($0, 1, pos-1)
|
||||
gsub(/^ +| +$/, "", name)
|
||||
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
||||
before[slave SUBSEP name] = val
|
||||
}
|
||||
next }
|
||||
{ if ($0 ~ /^\[/) slave=$0
|
||||
else if ($0 ~ /=/) {
|
||||
pos = index($0, "=")
|
||||
name = substr($0, 1, pos-1); gsub(/^ +| +$/, "", name)
|
||||
val = substr($0, pos+1); gsub(/^ +| +$/, "", val)
|
||||
b = before[slave SUBSEP name]+0
|
||||
d = val+0 - b
|
||||
if (d != 0) printf " %-8s %-30s %12s -> %12s delta=%d\n", slave, name, b, val, d
|
||||
} }
|
||||
' "$1" "$2"
|
||||
}
|
||||
|
||||
{
|
||||
echo "=== LACP retransmit cause investigation ($ROLE) ==="
|
||||
echo "Host: $HOST Role: $ROLE"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Bond: $BOND"
|
||||
echo ""
|
||||
echo "--- bond0 hash + driver ---"
|
||||
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac" /proc/net/bonding/$BOND
|
||||
# shellcheck disable=SC2013 # interface names contain no spaces; word-splitting is safe here
|
||||
for s in $(awk '/^Slave Interface:/{print $3}' /proc/net/bonding/$BOND); do
|
||||
drv=$(ethtool -i "$s" 2>/dev/null | awk -F: '/^driver:/{print $2}' | sed 's/^ *//')
|
||||
speed=$(cat "/sys/class/net/$s/speed" 2>/dev/null)
|
||||
ring=$(ethtool -g "$s" 2>/dev/null | awk '/RX:/{print $2; exit}')
|
||||
printf " %-8s driver=%-20s speed=%-6s current RX ring=%s\n" "$s" "$drv" "$speed" "$ring"
|
||||
done
|
||||
echo ""
|
||||
} | tee "$LOG"
|
||||
|
||||
echo "--- BEFORE snapshots ---"
|
||||
nic_snapshot > "$TMP/nic_before"
|
||||
softnet_snapshot > "$TMP/softnet_before"
|
||||
tcp_snapshot > "$TMP/tcp_before"
|
||||
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
echo "[receiver] starting iperf3 -s -1 on $PEER_RX_IP ..."
|
||||
pkill -x iperf3 2>/dev/null; sleep 0.5
|
||||
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
||||
SRV_PID=$!
|
||||
for _ in $(seq 1 60); do kill -0 "$SRV_PID" 2>/dev/null || break; sleep 1; done
|
||||
wait "$SRV_PID" 2>/dev/null
|
||||
else
|
||||
echo "[sender] waiting 3s for receiver to listen, then running iperf3 -P $STREAMS -t $DURATION ..."
|
||||
sleep 3
|
||||
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
||||
> "$TMP/iperf.out" 2>&1
|
||||
fi
|
||||
|
||||
echo "--- AFTER snapshots ---"
|
||||
nic_snapshot > "$TMP/nic_after"
|
||||
softnet_snapshot > "$TMP/softnet_after"
|
||||
tcp_snapshot > "$TMP/tcp_after"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "--- iperf3 summary ---"
|
||||
grep -E '\[SUM\].*(sender|receiver)' "$TMP/iperf.out" | tail -2
|
||||
echo ""
|
||||
echo "--- TCP SNMP deltas (/proc/net/snmp) ---"
|
||||
# Show only the counters that changed
|
||||
paste "$TMP/tcp_before" "$TMP/tcp_after" \
|
||||
| awk '{
|
||||
a=$3; b=$(NF)
|
||||
if (a+0 != b+0) printf " %-22s %12s -> %12s delta=%d\n", $1, a, b, b-a
|
||||
}'
|
||||
echo ""
|
||||
echo "--- softnet_stat deltas (look for non-zero dropped_delta/squeezed_delta) ---"
|
||||
softnet_delta "$TMP/softnet_before" "$TMP/softnet_after"
|
||||
echo ""
|
||||
echo "--- NIC counter deltas (only non-zero) ---"
|
||||
nic_delta "$TMP/nic_before" "$TMP/nic_after" "$HOST"
|
||||
echo ""
|
||||
echo "--- raw iperf3 tail (last 12 lines) ---"
|
||||
tail -12 "$TMP/iperf.out"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " HOW TO READ THIS"
|
||||
echo "==================================================================="
|
||||
echo " - If NIC rx_dropped / rx_missed_errors / rx_fifo_errors climbed:"
|
||||
echo " -> ring buffers too small. Fix: ethtool -G \$IF rx 4096 (or max)."
|
||||
echo " - If softnet_stat 'dropped' or 'squeezed' climbed on any CPU:"
|
||||
echo " -> softirq starvation. Fix: increase net.core.netdev_budget,"
|
||||
echo " check IRQ affinity, consider RPS."
|
||||
echo " - If NEITHER of the above climbed but TCP RetransSegs did:"
|
||||
echo " -> the retransmits are LACP reordering (out-of-order segments"
|
||||
echo " triggering fast retransmit), NOT packet loss. NOT fixable."
|
||||
echo "==================================================================="
|
||||
} | tee -a "$LOG"
|
||||
|
||||
echo ""
|
||||
echo "Log: $LOG"
|
||||
Executable
+153
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# lacp-rx-distribution.sh (HOST-NATIVE version)
|
||||
#
|
||||
# Runs ON a host (tsys6 or tsys7). Auto-detects role by hostname:
|
||||
# - On tsys6 (receiver): starts iperf3 -s -1 (one-shot), snapshots local
|
||||
# bond0 slave RX counters before+after, writes verdict + log.
|
||||
# - On tsys7 (sender): snapshots local bond0 slave TX counters before+
|
||||
# after, runs iperf3 -c <peer> -P 8 -t 12.
|
||||
#
|
||||
# Each side writes only its own counters. No inter-host SSH required.
|
||||
# iperf3 -s -1 (one-shot) handles client/server coordination.
|
||||
#
|
||||
# Output: /root/lacp-rx-{receiver,sender}.log
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
DURATION="${LACP_TEST_SECS:-12}"
|
||||
STREAMS="${LACP_STREAMS:-8}"
|
||||
PEER_RX_IP="10.100.100.6" # tsys6 storage IP (receiver)
|
||||
PEER_TX_IP="10.100.100.7" # tsys7 storage IP (sender)
|
||||
LOG_DIR="/root"
|
||||
BOND="bond0"
|
||||
|
||||
# Snapshot per-slave byte counters from $BOND. Output: "<slave> rx tx\n" sorted.
|
||||
snapshot() {
|
||||
awk '/^Slave Interface:/{print $3}' "/proc/net/bonding/$BOND" | while read -r s; do
|
||||
[ -n "$s" ] || continue
|
||||
rx=$(cat "/sys/class/net/$s/statistics/rx_bytes" 2>/dev/null || echo 0)
|
||||
tx=$(cat "/sys/class/net/$s/statistics/tx_bytes" 2>/dev/null || echo 0)
|
||||
printf '%s %s %s\n' "$s" "$rx" "$tx"
|
||||
done | sort
|
||||
}
|
||||
|
||||
# Compute per-slave deltas. $1=label, $2=col(2=rx,3=tx), $3=before, $4=after.
|
||||
analyze() {
|
||||
local label="$1" col="$2" before="$3" after="$4"
|
||||
join "$before" "$after" | awk -v col="$col" -v lbl="$label" '
|
||||
{
|
||||
slave=$1
|
||||
b = (col==2 ? $2 : $3)+0
|
||||
a = (col==2 ? $4 : $5)+0
|
||||
d = a - b; if (d < 0) d = 0
|
||||
delta[slave]=d; bef[slave]=b; aft[slave]=a
|
||||
order[++n]=slave; total+=d
|
||||
}
|
||||
END {
|
||||
printf "\n--- %s (bytes) ---\n", lbl
|
||||
printf " %-12s %14s %14s %14s %8s\n", "SLAVE", "BEFORE", "AFTER", "DELTA", "PCT"
|
||||
for (i=1;i<=n;i++){
|
||||
s=order[i]
|
||||
pct = (total>0 ? 100*delta[s]/total : 0)
|
||||
printf " %-12s %14d %14d %14d %7.1f%%\n", s, bef[s], aft[s], delta[s], pct
|
||||
}
|
||||
printf " %-12s %14s %14s %14d %8s\n", "TOTAL", "", "", total, "100.0%"
|
||||
}'
|
||||
}
|
||||
|
||||
# Dominant-slave pct for the verdict line. $1=col, $2=before, $3=after.
|
||||
dominant() {
|
||||
join "$2" "$3" | awk -v col="$1" '
|
||||
{ b=(col==2?$2:$3)+0; a=(col==2?$4:$5)+0; d=a-b; if(d<0)d=0; tot+=d; delta[$1]=d }
|
||||
END { m=0; ms=""; for (s in delta){ if (delta[s]>m){m=delta[s]; ms=s} }
|
||||
printf "%.1f %s", (tot>0?100*m/tot:0), ms }'
|
||||
}
|
||||
|
||||
HOST="$(uname -n)"
|
||||
HOST="${HOST%%.*}"
|
||||
case "$HOST" in
|
||||
*tsys6) ROLE="receiver"; LOG="$LOG_DIR/lacp-rx-receiver.log" ;;
|
||||
*tsys7) ROLE="sender"; LOG="$LOG_DIR/lacp-rx-sender.log" ;;
|
||||
*) echo "ERROR: not running on tsys6 or tsys7 (hostname=$HOST)"; exit 2 ;;
|
||||
esac
|
||||
|
||||
TMP="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
{
|
||||
echo "=== LACP per-slave distribution ($ROLE) ==="
|
||||
echo "Host: $HOST Role: $ROLE"
|
||||
echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
echo "Bond: $BOND"
|
||||
echo ""
|
||||
echo "--- $BOND state ---"
|
||||
grep -E "Bonding Mode|Transmit Hash|Number of ports|Partner Mac|Slave Interface|Link" \
|
||||
"/proc/net/bonding/$BOND" 2>&1
|
||||
echo ""
|
||||
} | tee "$LOG"
|
||||
|
||||
snapshot > "$TMP/before"
|
||||
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
echo "[receiver] starting one-shot iperf3 server on $PEER_RX_IP ..."
|
||||
pkill -x iperf3 2>/dev/null; sleep 0.5
|
||||
nohup iperf3 -s -1 -B "$PEER_RX_IP" > "$TMP/iperf.out" 2>&1 &
|
||||
SRV_PID=$!
|
||||
# Wait for the server to be ready (brief), then wait for it to exit
|
||||
# (one-shot server exits after serving one client).
|
||||
for _ in $(seq 1 60); do
|
||||
kill -0 "$SRV_PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
wait "$SRV_PID" 2>/dev/null
|
||||
echo "[receiver] iperf3 server finished."
|
||||
else
|
||||
# sender: wait briefly for receiver to be listening, then run client.
|
||||
echo "[sender] waiting 3s for receiver to listen, then running iperf3 client..."
|
||||
sleep 3
|
||||
iperf3 -c "$PEER_RX_IP" -B "$PEER_TX_IP" -P "$STREAMS" -t "$DURATION" -l 128k -O 2 \
|
||||
> "$TMP/iperf.out" 2>&1
|
||||
echo "[sender] iperf3 client finished (exit=$?)."
|
||||
fi
|
||||
|
||||
snapshot > "$TMP/after"
|
||||
|
||||
{
|
||||
echo ""
|
||||
echo "--- iperf3 output (raw tail) ---"
|
||||
tail -25 "$TMP/iperf.out"
|
||||
echo ""
|
||||
if [ "$ROLE" = "receiver" ]; then
|
||||
analyze "Receiver RX (THE key column)" 2 "$TMP/before" "$TMP/after"
|
||||
dom="$(dominant 2 "$TMP/before" "$TMP/after")"
|
||||
pct=${dom%% *}; top=${dom##* }
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
|
||||
echo " VERDICT: SWITCH NOT distributing across $HOST's two ports"
|
||||
echo " Dominant slave '$top' = ${pct}% of RX -> hash not effective"
|
||||
echo " on this LAG. Action: verify/bounce pfv-r3-tor-stor port-channel."
|
||||
else
|
||||
echo " VERDICT: switch IS distributing (top slave '$top' = ${pct}%)."
|
||||
echo " Cap is host-side: softirq/CPU/bridge/NIC coalescing."
|
||||
fi
|
||||
echo "==================================================================="
|
||||
else
|
||||
analyze "Sender TX (control: should split if host hash=layer3+4)" 3 "$TMP/before" "$TMP/after"
|
||||
dom="$(dominant 3 "$TMP/before" "$TMP/after")"
|
||||
pct=${dom%% *}; top=${dom##* }
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
if awk -v p="$pct" 'BEGIN{exit !(p>=90)}'; then
|
||||
echo " VERDICT: host TX NOT distributing (top slave '$top' = ${pct}%)."
|
||||
echo " Host xmit_hash_policy is NOT effective despite /proc/net/bonding."
|
||||
else
|
||||
echo " VERDICT: host TX distributing OK (top slave '$top' = ${pct}%)."
|
||||
fi
|
||||
echo "==================================================================="
|
||||
fi
|
||||
} | tee -a "$LOG"
|
||||
|
||||
echo ""
|
||||
echo "Log written: $LOG"
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/bin/bash
|
||||
# Read-only fleet drift probe — gathers package/tooling/config state for
|
||||
# consistency comparison across hosts. Writes only stdout.
|
||||
set -u
|
||||
echo "===== DRIFT PROBE: $(hostname -s) $(date -u +%FT%TZ) ====="
|
||||
echo
|
||||
echo "##### OS / kernel / PVE #####"
|
||||
pveversion 2>&1
|
||||
cat /etc/debian_version 2>&1
|
||||
echo
|
||||
echo "##### Installed key packages (versions) #####"
|
||||
dpkg-query -W -f='${Package}\t${Version}\n' 2>/dev/null | grep -iE 'lldpd|lldpad|smartmontools|nfs-common|nfs-kernel-server|iperf3|tcpdump|htop|rsyslog|qemu-guest-agent|snmpd|snmp|net-tools|ethtool|sysstat|ioping|fio|nvme-cli|conman|ser2net|nut-server|nut-client|tuned-adm|tuned' 2>/dev/null | sort
|
||||
echo
|
||||
echo "##### lldpcli present? #####"
|
||||
command -v lldpcli >/dev/null 2>&1 && lldpcli -v 2>&1 | head -1 || echo "lldpcli: NOT INSTALLED"
|
||||
echo
|
||||
echo "##### lldpd service #####"
|
||||
systemctl is-active lldpd 2>&1 || true
|
||||
systemctl is-enabled lldpd 2>&1 || true
|
||||
echo
|
||||
echo "##### smartmontools service #####"
|
||||
systemctl is-active smartd 2>&1 || echo "smartd: inactive/disabled"
|
||||
systemctl is-enabled smartd 2>&1 || true
|
||||
echo
|
||||
echo "##### snmpd service #####"
|
||||
systemctl is-active snmpd 2>&1 || echo "snmpd: not running"
|
||||
systemctl is-enabled snmpd 2>&1 || true
|
||||
echo
|
||||
echo "##### rsyslog #####"
|
||||
systemctl is-active rsyslog 2>&1 || echo "rsyslog: not running"
|
||||
echo
|
||||
echo "##### tuned profile #####"
|
||||
tuned-adm active 2>&1 | head -2 || echo "tuned: not available"
|
||||
echo
|
||||
echo "##### sshd config (key settings) #####"
|
||||
grep -E '^(PermitRootLogin|PasswordAuthentication|PubkeyAuthentication|KbdInteractiveAuthentication|Port |PermitEmptyPasswords)' /etc/ssh/sshd_config 2>/dev/null || echo "(defaults)"
|
||||
grep -E '^(PermitRootLogin|PasswordAuthentication|PubkeyAuthentication|KbdInteractiveAuthentication|Port |PermitEmptyPasswords)' /etc/ssh/sshd_config.d/*.conf 2>/dev/null || true
|
||||
echo
|
||||
echo "##### authorized_keys (root) — count + key types #####"
|
||||
if [ -f /root/.ssh/authorized_keys ]; then
|
||||
echo " total keys: $(wc -l < /root/.ssh/authorized_keys)"
|
||||
awk '{print " "$1}' /root/.ssh/authorized_keys | sort | uniq -c
|
||||
else
|
||||
echo " NO authorized_keys for root"
|
||||
fi
|
||||
echo
|
||||
echo "##### ssh host key fingerprints #####"
|
||||
for f in /etc/ssh/ssh_host_*_key.pub; do
|
||||
[ -f "$f" ] && ssh-keygen -lf "$f" 2>/dev/null
|
||||
done
|
||||
echo
|
||||
echo "##### sysctl tuning (key values) #####"
|
||||
echo " net.core.rmem_max=$(cat /proc/sys/net/core/rmem_max 2>/dev/null)"
|
||||
echo " net.core.wmem_max=$(cat /proc/sys/net/core/wmem_max 2>/dev/null)"
|
||||
echo " net.ipv4.tcp_congestion_control=$(cat /proc/sys/net/ipv4/tcp_congestion_control 2>/dev/null)"
|
||||
echo " net.ipv4.tcp_rmem=$(cat /proc/sys/net/ipv4/tcp_rmem 2>/dev/null)"
|
||||
echo " net.ipv4.tcp_wmem=$(cat /proc/sys/net/ipv4/tcp_wmem 2>/dev/null)"
|
||||
echo " net.ipv4.tcp_max_syn_backlog=$(cat /proc/sys/net/ipv4/tcp_max_syn_backlog 2>/dev/null)"
|
||||
echo " net.core.netdev_max_backlog=$(cat /proc/sys/net/core/netdev_max_backlog 2>/dev/null)"
|
||||
echo " vm.swappiness=$(cat /proc/sys/vm/swappiness 2>/dev/null)"
|
||||
echo
|
||||
echo "##### CPU governor #####"
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "no cpufreq driver"
|
||||
echo
|
||||
echo "##### beszel agent? #####"
|
||||
systemctl is-active beszel-hub 2>/dev/null || systemctl is-active beszel-agent 2>/dev/null || echo "beszel: not installed"
|
||||
echo
|
||||
echo "##### /root/bin or custom scripts present? #####"
|
||||
# shellcheck disable=SC2012 # diagnostic listing, not for processing
|
||||
find /root/bin/ -maxdepth 1 -type f 2>/dev/null | head -10 || echo "(no /root/bin)"
|
||||
find /root/ -maxdepth 1 -name '*.sh' -type f 2>/dev/null | head -10 || echo "(no /root/*.sh)"
|
||||
echo
|
||||
echo "##### mount options (noatime check on root) #####"
|
||||
mount | grep ' / ' | head -1
|
||||
echo
|
||||
echo "##### NFS server threads (if NFS server) #####"
|
||||
grep -E '^[[:space:]]*RPCNFSDCOUNT' /etc/default/nfs-kernel-server 2>/dev/null || grep -E 'RPCNFSDCOUNT' /etc/default/nfs-kernel-server 2>/dev/null || echo "(default, not a NFS server or default threads)"
|
||||
echo
|
||||
echo "===== END $(hostname -s) ====="
|
||||
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# probe-network.sh
|
||||
#
|
||||
# READ-ONLY network + NFS ground-truth probe. Writes only stdout.
|
||||
# Run on any Proxmox host to inventory its NICs, bonds, LLDP neighbors,
|
||||
# NFS client mounts (including nconnect), ethtool link state, and error
|
||||
# counters. No hardcoded values — fully portable.
|
||||
#
|
||||
# Usage (via tests/remote.sh):
|
||||
# PROX_HOST=pfv-tsys6 bash tests/remote.sh prox-file perf/scripts/probe-network.sh
|
||||
###############################################################################
|
||||
set -u
|
||||
echo "===== HOST: $(hostname -s) $(date -u +%FT%TZ) ====="
|
||||
echo
|
||||
echo "##### ip -br link #####"
|
||||
ip -br link 2>&1
|
||||
echo
|
||||
echo "##### ip -br addr #####"
|
||||
ip -br addr 2>&1
|
||||
echo
|
||||
echo "##### /etc/network/interfaces #####"
|
||||
cat /etc/network/interfaces 2>&1
|
||||
echo
|
||||
echo "##### bond0 state (if present) #####"
|
||||
if [ -r /proc/net/bonding/bond0 ]; then
|
||||
cat /proc/net/bonding/bond0 2>&1
|
||||
else
|
||||
echo "(no bond0)"
|
||||
fi
|
||||
echo
|
||||
echo "##### ethtool per physical NIC #####"
|
||||
for nic in /sys/class/net/*; do
|
||||
nic=$(basename "$nic")
|
||||
case "$nic" in lo|bond*|br*|venet*|veth*|docker*|tap*|vnet*|fw*) continue;; esac
|
||||
echo "--- ethtool $nic ---"
|
||||
ethtool "$nic" 2>&1 | grep -iE 'Speed|Duplex|Port|Link|Supported link modes|Advertising|Auto-neg|Settings' || echo "(ethtool failed for $nic)"
|
||||
done
|
||||
echo
|
||||
echo "##### lldpcli (if installed) #####"
|
||||
if command -v lldpcli >/dev/null 2>&1; then
|
||||
echo "--- lldpcli show neighbors ---"
|
||||
lldpcli show neighbors 2>&1
|
||||
echo
|
||||
echo "--- lldpcli show interfaces ---"
|
||||
lldpcli show interfaces 2>&1
|
||||
echo
|
||||
echo "--- lldpcli show chassis ---"
|
||||
lldpcli show chassis 2>&1
|
||||
else
|
||||
echo "(lldpcli not installed)"
|
||||
fi
|
||||
echo
|
||||
echo "##### lldpd / lldpad service #####"
|
||||
systemctl is-active lldpd 2>&1 || true
|
||||
systemctl is-enabled lldpd 2>&1 || true
|
||||
echo
|
||||
echo "##### NFS mounts (mount | grep nfs) #####"
|
||||
mount | grep -i nfs 2>&1 || echo "(no nfs mounts)"
|
||||
echo
|
||||
echo "##### mount nconnect detail (nfsstat -m) #####"
|
||||
nfsstat -m 2>&1
|
||||
echo
|
||||
echo "##### storage.cfg NFS stanzas (options) #####"
|
||||
grep -A3 '^nfs:' /etc/pve/storage.cfg 2>&1
|
||||
echo
|
||||
echo "##### ip route #####"
|
||||
ip route 2>&1
|
||||
echo
|
||||
echo "##### ethtool -S bond slaves (key counters) #####"
|
||||
if [ -r /proc/net/bonding/bond0 ]; then
|
||||
# shellcheck disable=SC2013 # intentional: extract NIC names from bonding info
|
||||
for nic in $(grep -oE 'eth[0-9]+|en[psx][a-z0-9]+' /proc/net/bonding/bond0 2>/dev/null | sort -u); do
|
||||
echo "--- ethtool -S $nic (errors) ---"
|
||||
ethtool -S "$nic" 2>/dev/null | grep -iE 'error|drop|discard|crc|pause|miss' || echo "(no error counters)"
|
||||
done
|
||||
fi
|
||||
echo
|
||||
echo "##### ip neigh (ARP table, reachable/stale) #####"
|
||||
ip neigh show 2>&1 | grep -vE ' FAILED|INCOMPLETE' | sort -t. -k4 -n
|
||||
echo
|
||||
echo "===== END $(hostname -s) ====="
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# probe-storage.sh
|
||||
#
|
||||
# READ-ONLY storage + disk ground-truth probe. Writes only stdout.
|
||||
# Run on any Proxmox host (or any Linux NFS server) to inventory its physical
|
||||
# disks, mounts, exports, SMART health, and Proxmox storage config.
|
||||
#
|
||||
# Portable: no hardcoded values. Uses only standard CLI tools + smartmontools.
|
||||
#
|
||||
# Usage (via tests/remote.sh):
|
||||
# PROX_HOST=pfv-tsys4 bash tests/remote.sh prox-file perf/scripts/probe-storage.sh
|
||||
#
|
||||
# Or directly on a host:
|
||||
# bash probe-storage.sh > storage-audit.txt
|
||||
###############################################################################
|
||||
set -u
|
||||
echo "===== HOST: $(hostname -s) $(date -u +%FT%TZ) ====="
|
||||
echo
|
||||
echo "##### lsblk (tree, with model/serial/size/type) #####"
|
||||
lsblk -o NAME,MAJ:MIN,SIZE,TYPE,MOUNTPOINT,MODEL,SERIAL,STATE,ROTA,TRAN,REV 2>&1
|
||||
echo
|
||||
echo "##### block devices by-id #####"
|
||||
for dev in /dev/disk/by-id/*; do
|
||||
[ -L "$dev" ] || continue
|
||||
case "$(basename "$dev")" in *part[0-9]*) continue;; esac
|
||||
ls -l "$dev"
|
||||
done 2>&1
|
||||
echo
|
||||
echo "##### nvme list (if any) #####"
|
||||
command -v nvme >/dev/null 2>&1 && nvme list 2>&1 || echo "(no nvme-cli or no nvme devices)"
|
||||
echo
|
||||
echo "##### blkid #####"
|
||||
blkid 2>&1
|
||||
echo
|
||||
echo "##### mounted filesystems #####"
|
||||
findmnt -o TARGET,SOURCE,FSTYPE,OPTIONS 2>&1
|
||||
echo
|
||||
echo "##### /etc/fstab #####"
|
||||
cat /etc/fstab 2>&1
|
||||
echo
|
||||
echo "##### /etc/exports (+ exports.d) #####"
|
||||
cat /etc/exports 2>&1
|
||||
for f in /etc/exports.d/*.exports; do [ -f "$f" ] && echo "--- $f ---" && cat "$f"; done 2>&1
|
||||
echo
|
||||
echo "##### df -h (all mounts) #####"
|
||||
df -h 2>&1
|
||||
echo
|
||||
echo "##### smartctl -a per block device #####"
|
||||
command -v smartctl >/dev/null 2>&1 || echo "(smartctl not installed)"
|
||||
for d in /dev/sd? /dev/nvme?n1; do
|
||||
[ -b "$d" ] || continue
|
||||
echo "----- smartctl -a $d -----"
|
||||
smartctl -a "$d" 2>&1 | grep -iE 'Device Model|Model Number|Serial|Firmware|User Capacity|Rotation Rate|Form Factor|SATA Version|NVMe|SMART overall|Reallocated|Pending|Uncorrect|Power On|Temperature|Media and Data Integrity' || true
|
||||
done
|
||||
echo
|
||||
echo "##### /etc/pve/storage.cfg #####"
|
||||
cat /etc/pve/storage.cfg 2>&1
|
||||
echo
|
||||
echo "##### pvesm status #####"
|
||||
pvesm status 2>&1
|
||||
echo
|
||||
echo "##### pvesm list per store #####"
|
||||
for s in $(pvesm status 2>/dev/null | awk 'NR>1 && $3>0 {print $1}'); do
|
||||
echo "--- pvesm list $s ---"
|
||||
pvesm list "$s" 2>&1 | head -40
|
||||
done
|
||||
echo
|
||||
echo "##### zpool status (if any) #####"
|
||||
command -v zpool >/dev/null 2>&1 && zpool status 2>&1 || echo "(no zfs)"
|
||||
echo
|
||||
echo "##### lvm: pvs/vgs/lvs #####"
|
||||
command -v pvs >/dev/null 2>&1 && { pvs 2>&1; echo; vgs 2>&1; echo; lvs 2>&1; } || echo "(no lvm tools)"
|
||||
echo
|
||||
echo "===== END $(hostname -s) ====="
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# Read-only DNS consistency checker for Proxmox VMs.
|
||||
# Checks that every running VM's name resolves to the IP it's actually using,
|
||||
# and that reverse DNS (PTR) matches the VM name. Mismatches are reported as bugs.
|
||||
set -u
|
||||
echo "===== VM DNS CONSISTENCY CHECK: $(hostname -s) $(date -u +%FT%TZ) ====="
|
||||
echo
|
||||
echo "##### Running VMs with IPs (via guest-agent or ARP) #####"
|
||||
|
||||
for vmid in $(qm list 2>/dev/null | awk 'NR>1 && $4=="running" {print $1}'); do
|
||||
name=$(qm config "$vmid" 2>/dev/null | awk '/^name:/{print $2}')
|
||||
# Get the VM's net0 config to determine the bridge
|
||||
net0=$(qm config "$vmid" 2>/dev/null | awk '/^net0:/{print $0}')
|
||||
echo ""
|
||||
echo "--- VMID $vmid: $name ---"
|
||||
echo " net0: $(echo "$net0" | sed 's/net0: //')"
|
||||
|
||||
# Try to get IP via guest-agent
|
||||
if qm config "$vmid" 2>/dev/null | grep -q 'agent:.*enabled=1\|^agent: 1'; then
|
||||
guest_ips=$(qm guest cmd "$vmid" network-get-interfaces 2>/dev/null)
|
||||
if [ -n "$guest_ips" ]; then
|
||||
echo " guest-agent IPs:"
|
||||
echo "$guest_ips" | python3 -c "
|
||||
import sys, json
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
for iface in data:
|
||||
name = iface.get('name','?')
|
||||
for addr in iface.get('ip-addresses',[]):
|
||||
ip = addr.get('ip-address','')
|
||||
typ = addr.get('ip-address-type','')
|
||||
if typ == 'ipv4' and not ip.startswith('127.'):
|
||||
print(f' {name}: {ip}')
|
||||
except: pass
|
||||
" 2>/dev/null
|
||||
else
|
||||
echo " guest-agent: no response"
|
||||
fi
|
||||
else
|
||||
echo " guest-agent: not enabled"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
echo "===== END $(hostname -s) ====="
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# run-lacp-retrans-cause.sh (workstation wrapper)
|
||||
#
|
||||
# Deploys scripts/lacp-retrans-cause.sh to BOTH hosts, runs them in the
|
||||
# right order, scps logs back, prints them.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
RECV="pfv-tsys6"; SEND="pfv-tsys7"
|
||||
LOCAL_LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/returned-logs/iperf"
|
||||
SCRIPT="lacp-retrans-cause.sh"
|
||||
LOCAL_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/scripts/${SCRIPT}"
|
||||
REMOTE_SCRIPT="/root/${SCRIPT}"
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " LACP retransmit-cause investigation"
|
||||
echo " Receiver: $RECV Sender: $SEND"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
echo "--- preflight ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
ssh "${SSH[@]}" "root@$h" \
|
||||
'command -v ethtool >/dev/null && e=OK || e=MISSING
|
||||
command -v iperf3 >/dev/null && i=OK || i=MISSING
|
||||
printf "ethtool=%s iperf3=%s host=%s\n" "$e" "$i" "$(uname -n)"' 2>&1 | head -1
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- deploy ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
||||
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
||||
&& echo "deployed" || echo "FAILED"
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- cleanup stale iperf3 ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "--- launching receiver on $RECV (background) ---"
|
||||
ssh "${SSH[@]}" "root@$RECV" \
|
||||
"nohup bash $REMOTE_SCRIPT > /root/lacp-retrans-receiver.console 2>&1 &" 2>/dev/null
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
||||
ssh "${SSH[@]}" "root@$SEND" "bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
||||
echo ""
|
||||
sleep 3
|
||||
|
||||
echo "--- fetching logs ---"
|
||||
for f in lacp-retrans-receiver.log lacp-retrans-receiver.console lacp-retrans-sender.log; do
|
||||
src=""
|
||||
case "$f" in
|
||||
*receiver*) src="$RECV" ;;
|
||||
*sender*) src="$SEND" ;;
|
||||
esac
|
||||
printf ' %-32s <- %s : ' "$f" "$src"
|
||||
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
||||
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
||||
else
|
||||
echo "MISSING"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
echo "==================================================================="
|
||||
echo " RECEIVER LOG ($RECV)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-retrans-receiver.log" 2>/dev/null || echo "(missing)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SENDER LOG ($SEND)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-retrans-sender.log" 2>/dev/null || echo "(missing)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Local copies in: $LOCAL_LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env bash
|
||||
###############################################################################
|
||||
# run-lacp-rx-distribution.sh (workstation wrapper)
|
||||
#
|
||||
# One-shot orchestrator: deploys scripts/lacp-rx-distribution.sh to BOTH
|
||||
# tsys6 (receiver) and tsys7 (sender), runs them in the right order, then
|
||||
# scps both logs back to returned-logs/iperf/.
|
||||
#
|
||||
# Run from the workstation:
|
||||
# bash scripts/run-lacp-rx-distribution.sh
|
||||
#
|
||||
# Idempotent: safe to re-run. iperf3 is killed on both hosts first.
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
|
||||
SSH=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 \
|
||||
-o StrictHostKeyChecking=accept-new)
|
||||
SCP=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
RECV="pfv-tsys6" # receiver (we care most about its RX split)
|
||||
SEND="pfv-tsys7" # sender (control: its TX split)
|
||||
LOCAL_LOG_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/returned-logs/iperf"
|
||||
SCRIPT="lacp-rx-distribution.sh"
|
||||
LOCAL_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/scripts/${SCRIPT}"
|
||||
REMOTE_SCRIPT="/root/${SCRIPT}"
|
||||
|
||||
mkdir -p "$LOCAL_LOG_DIR"
|
||||
|
||||
echo "==================================================================="
|
||||
echo " LACP per-slave RX/TX distribution test"
|
||||
echo " Receiver: $RECV (10.100.100.6) Sender: $SEND (10.100.100.7)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
# 0. Preflight: confirm SSH and iperf3 on both hosts
|
||||
echo "--- preflight (ssh + iperf3 + bond0) ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
ssh "${SSH[@]}" "root@$h" \
|
||||
'command -v iperf3 >/dev/null && ip=$(command -v iperf3) || ip=MISSING
|
||||
[ -r /proc/net/bonding/bond0 ] && b=OK || b=NO-BOND0
|
||||
printf "iperf3=%s bond0=%s host=%s\n" "$ip" "$b" "$(uname -n)"' \
|
||||
2>&1 | head -1
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 1. Copy the script to both hosts + chmod
|
||||
echo "--- deploy $SCRIPT to both hosts ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
printf ' %-12s ' "$h"
|
||||
scp "${SCP[@]}" "$LOCAL_SCRIPT" "root@$h:$REMOTE_SCRIPT" >/dev/null 2>&1 \
|
||||
&& ssh "${SSH[@]}" "root@$h" "chmod +x $REMOTE_SCRIPT" \
|
||||
&& echo "deployed + chmod +x" \
|
||||
|| echo "DEPLOY FAILED"
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 2. Kill any stale iperf3 on both hosts
|
||||
echo "--- cleanup stale iperf3 ---"
|
||||
for h in "$RECV" "$SEND"; do
|
||||
ssh "${SSH[@]}" "root@$h" 'pkill -x iperf3 2>/dev/null; true' 2>/dev/null
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 3. Start RECEIVER in background (one-shot server, writes /root/lacp-rx-receiver.log)
|
||||
echo "--- starting receiver on $RECV (background) ---"
|
||||
ssh "${SSH[@]}" "root@$RECV" \
|
||||
"nohup bash $REMOTE_SCRIPT > /root/lacp-rx-receiver.console 2>&1 &" 2>/dev/null
|
||||
echo " receiver launched; waiting 5s for it to start iperf3 -s -1 ..."
|
||||
sleep 5
|
||||
echo ""
|
||||
|
||||
# 4. Run SENDER (foreground; ~15s with the default 12s test + 3s pre-sleep)
|
||||
echo "--- running sender on $SEND (foreground, ~20s) ---"
|
||||
ssh "${SSH[@]}" "root@$SEND" \
|
||||
"bash $REMOTE_SCRIPT" 2>&1 | sed 's/^/ [sender] /'
|
||||
echo ""
|
||||
|
||||
# 5. Give receiver a moment to finish writing its log
|
||||
sleep 3
|
||||
|
||||
# 6. Fetch logs back
|
||||
echo "--- fetching logs ---"
|
||||
for f in lacp-rx-receiver.log lacp-rx-receiver.console lacp-rx-sender.log; do
|
||||
src=""
|
||||
case "$f" in
|
||||
lacp-rx-receiver*) src="$RECV" ;;
|
||||
lacp-rx-sender*) src="$SEND" ;;
|
||||
esac
|
||||
printf ' %-28s <- %s : ' "$f" "$src"
|
||||
if scp "${SCP[@]}" "root@$src:/root/$f" "$LOCAL_LOG_DIR/$f" >/dev/null 2>&1; then
|
||||
echo "OK ($(wc -c < "$LOCAL_LOG_DIR/$f" 2>/dev/null) bytes)"
|
||||
else
|
||||
echo "MISSING"
|
||||
fi
|
||||
done
|
||||
echo ""
|
||||
|
||||
# 7. Show the receiver log (the decisive one)
|
||||
echo "==================================================================="
|
||||
echo " RECEIVER LOG ($RECV — the decisive side)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-rx-receiver.log" 2>/dev/null || echo "(no log)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " SENDER LOG ($SEND — control)"
|
||||
echo "==================================================================="
|
||||
cat "$LOCAL_LOG_DIR/lacp-rx-sender.log" 2>/dev/null || echo "(no log)"
|
||||
echo ""
|
||||
echo "==================================================================="
|
||||
echo " Local copies in: $LOCAL_LOG_DIR/"
|
||||
echo "==================================================================="
|
||||
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
snmp-switch-audit.py — READ-ONLY switch inventory via SNMP.
|
||||
|
||||
Gathers interface status/speed/errors, LLDP neighbor topology, port-channel
|
||||
(LAG) membership, and VLAN membership from any SNMPv2c-capable switch.
|
||||
Designed for Dell/Radlan (Neyland) and standard IF/LLDP/Q-BRIDGE MIB switches,
|
||||
but works on any SNMP-manageable device.
|
||||
|
||||
All parameters configurable via env vars or CLI flags so this works on any
|
||||
network:
|
||||
|
||||
SNMP_COMMUNITY SNMPv2c community string (env, default: public)
|
||||
SWITCH_IPS space-separated switch IPs (env, or pass as args)
|
||||
OUTPUT_DIR where to write per-switch (env, default: returned-logs/snmp)
|
||||
|
||||
Usage:
|
||||
SNMP_COMMUNITY=kn3lmgmt SWITCH_IPS="192.168.0.9 192.168.0.12" \\
|
||||
python3 snmp-switch-audit.py
|
||||
|
||||
# or pass IPs as positional args:
|
||||
SNMP_COMMUNITY=kn3lmgmt python3 snmp-switch-audit.py 192.168.0.9 192.168.0.12
|
||||
|
||||
Requires: pysnmp (pip install pysnmp) or net-snmp utils (snmpwalk) on PATH.
|
||||
Outputs: per-switch JSON + human-readable text in OUTPUT_DIR.
|
||||
|
||||
Read-only: sends only SNMP GET/GETNEXT/GETBULK. Never SETs anything.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
COMMUNITY = os.environ.get("SNMP_COMMUNITY", "public")
|
||||
OUTPUT_DIR = os.environ.get("OUTPUT_DIR",
|
||||
os.path.join(os.path.dirname(__file__), "..",
|
||||
"..", "returned-logs", "snmp"))
|
||||
|
||||
# OID constants
|
||||
OID_SYSDESCR = "1.3.6.1.2.1.1.1.0"
|
||||
OID_SYSNAME = "1.3.6.1.2.1.1.5.0"
|
||||
OID_IF_NAME = "1.3.6.1.2.1.31.1.1.1.1"
|
||||
OID_IF_SPEED = "1.3.6.1.2.1.2.2.1.5"
|
||||
OID_IF_OPER = "1.3.6.1.2.1.2.2.1.8"
|
||||
OID_IF_INERR = "1.3.6.1.2.1.2.2.1.14"
|
||||
OID_IF_OUTERR = "1.3.6.1.2.1.2.2.1.20"
|
||||
OID_IF_INOCT = "1.3.6.1.2.1.31.1.1.1.6"
|
||||
OID_IF_OUTOCT = "1.3.6.1.2.1.31.1.1.1.10"
|
||||
OID_LACP_LAG = "1.2.840.10006.300.43.1.1.1.1"
|
||||
OID_LLDP_REM_PORT = "1.0.8802.1.1.2.1.4.1.1.7"
|
||||
OID_LLDP_REM_SYSNAME = "1.0.8802.1.1.2.1.4.1.1.9"
|
||||
OID_LLDP_REM_CHASSIS = "1.0.8802.1.1.2.1.4.1.1.6"
|
||||
OID_LLDP_REM_LOCALPORT = "1.0.8802.1.1.2.1.4.1.1.3"
|
||||
OID_QBRIDGE_VLAN = "1.3.6.1.2.1.17.7.1.4.3.1.1"
|
||||
|
||||
|
||||
def snmpget(ip, oid):
|
||||
"""Single SNMP GET, returns string value or None."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["snmpget", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
if r.returncode == 0 and r.stdout.strip():
|
||||
return r.stdout.strip().strip('"')
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def snmpwalk(ip, oid):
|
||||
"""SNMP BULKWALK, returns dict of ifIndex -> value."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["snmpbulkwalk", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
if r.returncode != 0:
|
||||
r = subprocess.run(
|
||||
["snmpwalk", "-Oqv", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return {}
|
||||
result = {}
|
||||
for i, line in enumerate(r.stdout.strip().split("\n"), 1):
|
||||
line = line.strip().strip('"')
|
||||
if line:
|
||||
result[i] = line
|
||||
return result
|
||||
|
||||
|
||||
def walk_indexed(ip, oid):
|
||||
"""SNMP walk preserving OID index. Returns dict: index_str -> value."""
|
||||
try:
|
||||
r = subprocess.run(
|
||||
["snmpbulkwalk", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
if r.returncode != 0:
|
||||
r = subprocess.run(
|
||||
["snmpwalk", "-v2c", "-c", COMMUNITY, ip, oid],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
except (subprocess.TimeoutExpired, FileNotFoundError):
|
||||
return {}
|
||||
result = {}
|
||||
for line in r.stdout.strip().split("\n"):
|
||||
m = re.search(r'(\d+)\s*=\s*(.+)', line)
|
||||
if not m:
|
||||
m = re.search(r'\.(\d+)\s*=\s*(.+)', line)
|
||||
if m:
|
||||
idx = m.group(1).split(".")[-1]
|
||||
val = m.group(2).strip()
|
||||
val = re.sub(r'^(INTEGER: |STRING: |Hex-STRING: |Gauge32: |Counter32: |Counter64: )', '', val)
|
||||
result[idx] = val
|
||||
return result
|
||||
|
||||
|
||||
def audit_switch(ip):
|
||||
"""Gather all data for one switch."""
|
||||
data = {"ip": ip}
|
||||
data["sysDescr"] = snmpget(ip, OID_SYSDESCR)
|
||||
data["sysName"] = snmpget(ip, OID_SYSNAME)
|
||||
if not data["sysDescr"]:
|
||||
return data
|
||||
|
||||
names = snmpwalk(ip, OID_IF_NAME)
|
||||
speeds = snmpwalk(ip, OID_IF_SPEED)
|
||||
oper = snmpwalk(ip, OID_IF_OPER)
|
||||
inerr = snmpwalk(ip, OID_IF_INERR)
|
||||
outerr = snmpwalk(ip, OID_IF_OUTERR)
|
||||
|
||||
interfaces = []
|
||||
for idx in sorted(names.keys()):
|
||||
if idx not in names:
|
||||
continue
|
||||
speed_raw = speeds.get(idx, "0")
|
||||
try:
|
||||
speed_mbps = int(re.sub(r'\D', '', str(speed_raw))) // 1000000
|
||||
except (ValueError, TypeError):
|
||||
speed_mbps = 0
|
||||
is_up = str(oper.get(idx, "0")).strip() == "1"
|
||||
interfaces.append({
|
||||
"ifIndex": idx,
|
||||
"name": names[idx],
|
||||
"speedMbps": speed_mbps,
|
||||
"up": is_up,
|
||||
"inErrors": inerr.get(idx, "0"),
|
||||
"outErrors": outerr.get(idx, "0"),
|
||||
})
|
||||
data["interfaces"] = interfaces
|
||||
|
||||
# LLDP neighbors
|
||||
rem_ports = walk_indexed(ip, OID_LLDP_REM_PORT)
|
||||
rem_sysnames = walk_indexed(ip, OID_LLDP_REM_SYSNAME)
|
||||
rem_chassis = walk_indexed(ip, OID_LLDP_REM_CHASSIS)
|
||||
rem_local = walk_indexed(ip, OID_LLDP_REM_LOCALPORT)
|
||||
lldp = []
|
||||
for idx in rem_ports:
|
||||
lldp.append({
|
||||
"localPort": rem_local.get(idx, "?"),
|
||||
"remotePort": rem_ports[idx],
|
||||
"remoteSysName": rem_sysnames.get(idx, ""),
|
||||
"remoteChassis": rem_chassis.get(idx, ""),
|
||||
})
|
||||
data["lldpNeighbors"] = lldp
|
||||
|
||||
# LACP LAG table
|
||||
lag_data = walk_indexed(ip, OID_LACP_LAG)
|
||||
data["lagTable"] = lag_data
|
||||
|
||||
# VLAN membership
|
||||
vlan_data = walk_indexed(ip, OID_QBRIDGE_VLAN)
|
||||
data["vlans"] = vlan_data
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def print_switch(data):
|
||||
"""Human-readable summary."""
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {data.get('sysName', data['ip'])} ({data['ip']})")
|
||||
print(f" {data.get('sysDescr', '?')}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
print(f"\n Active ports (UP only):")
|
||||
print(f" {'Port':<12} {'Speed':>10} {'InErrors':>10} {'OutErrors':>10}")
|
||||
print(f" {'-'*12} {'-'*10} {'-'*10} {'-'*10}")
|
||||
for iface in data.get("interfaces", []):
|
||||
if iface["up"]:
|
||||
print(f" {iface['name']:<12} {iface['speedMbps']:>8}Mb "
|
||||
f"{iface['inErrors']:>10} {iface['outErrors']:>10}")
|
||||
|
||||
err_ports = [i for i in data.get("interfaces", [])
|
||||
if i["up"] and (int(i["inErrors"] or 0) > 0
|
||||
or int(i["outErrors"] or 0) > 0)]
|
||||
if err_ports:
|
||||
print(f"\n *** PORTS WITH ERRORS ***")
|
||||
for p in err_ports:
|
||||
print(f" {p['name']}: inErr={p['inErrors']} outErr={p['outErrors']}")
|
||||
|
||||
if data.get("lldpNeighbors"):
|
||||
print(f"\n LLDP neighbors:")
|
||||
for n in data["lldpNeighbors"]:
|
||||
sysname = n.get("remoteSysName", "") or "(unknown)"
|
||||
print(f" local={n['localPort']:<6} remote={n['remotePort']:<20} {sysname}")
|
||||
|
||||
if data.get("lagTable"):
|
||||
print(f"\n LACP/LAG table entries: {len(data['lagTable'])}")
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description="READ-ONLY SNMP switch audit (portable, config-driven)")
|
||||
ap.add_argument("switches", nargs="*",
|
||||
help="switch IPs (env: SWITCH_IPS)")
|
||||
ap.add_argument("--community", default=COMMUNITY,
|
||||
help=f"SNMPv2c community (env: SNMP_COMMUNITY, default: {COMMUNITY})")
|
||||
ap.add_argument("--output", default=OUTPUT_DIR,
|
||||
help=f"output dir (env: OUTPUT_DIR)")
|
||||
args = ap.parse_args()
|
||||
|
||||
community = args.community
|
||||
|
||||
ips = args.switches
|
||||
if not ips:
|
||||
env_ips = os.environ.get("SWITCH_IPS", "")
|
||||
ips = env_ips.split()
|
||||
|
||||
if not ips:
|
||||
ap.error("no switch IPs provided (pass as args or set SWITCH_IPS)")
|
||||
|
||||
os.makedirs(args.output, exist_ok=True)
|
||||
all_data = []
|
||||
|
||||
for ip in ips:
|
||||
globals()["COMMUNITY"] = community
|
||||
data = audit_switch(ip.strip())
|
||||
all_data.append(data)
|
||||
print_switch(data)
|
||||
outpath = os.path.join(args.output, f"switch-{ip}.json")
|
||||
with open(outpath, "w") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
print(f"\n -> {outpath}")
|
||||
|
||||
combined = os.path.join(args.output, "switches-all.json")
|
||||
with open(combined, "w") as f:
|
||||
json.dump(all_data, f, indent=2)
|
||||
print(f"\n Combined: {combined}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/bin/bash
|
||||
# Lint wrapper - permanent wrapper to lint every shell script in this project.
|
||||
#
|
||||
# Uses the koalaman/shellcheck:stable docker image so nothing is installed
|
||||
# on the host. Run from anywhere; lints scripts/ and any .sh under switches/.
|
||||
#
|
||||
# Usage:
|
||||
# ./shellcheck.sh # tty output, all scripts
|
||||
# ./shellcheck.sh --fix-info # treat style notes as non-blocking (default)
|
||||
# ./shellcheck.sh --strict # exit non-zero on ANY finding (notes too)
|
||||
# ./shellcheck.sh scripts/assess.sh # lint a single file
|
||||
set -u
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
||||
IMAGE="koalaman/shellcheck:stable"
|
||||
STRICT=0
|
||||
TARGETS=()
|
||||
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--strict) STRICT=1 ;;
|
||||
--fix-info) STRICT=0 ;;
|
||||
-h|--help)
|
||||
sed -n '2,12p' "$0"; exit 0 ;;
|
||||
*) TARGETS+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Default targets: everything in scripts/, plus any .cmds is NOT shell - skip.
|
||||
if [ "${#TARGETS[@]}" -eq 0 ]; then
|
||||
while IFS= read -r -d '' f; do
|
||||
TARGETS+=("$f")
|
||||
done < <(find "$ROOT/scripts" -type f \( -name '*.sh' -o -name 'collect-*' -o -name 'assess*' \) -print0 2>/dev/null)
|
||||
fi
|
||||
|
||||
if [ "${#TARGETS[@]}" -eq 0 ]; then
|
||||
echo "no shell scripts found to lint" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Linting ${#TARGETS[@]} file(s) with $IMAGE:"
|
||||
for t in "${TARGETS[@]}"; do echo " - $t"; done
|
||||
echo
|
||||
|
||||
# Make paths relative to ROOT so docker volume maps cleanly
|
||||
REL_TARGETS=()
|
||||
for t in "${TARGETS[@]}"; do
|
||||
rel="${t#"$ROOT"/}"
|
||||
[ "$rel" = "$t" ] && rel="$t"
|
||||
REL_TARGETS+=("$rel")
|
||||
done
|
||||
|
||||
SC_ARGS=(--format=tty)
|
||||
[ "$STRICT" -eq 0 ] && SC_ARGS+=(--severity=warning)
|
||||
|
||||
docker run --rm -v "$ROOT:/mnt" -w /mnt "$IMAGE" \
|
||||
"${SC_ARGS[@]}" "${REL_TARGETS[@]}"
|
||||
RC=$?
|
||||
|
||||
if [ "$STRICT" -eq 1 ]; then
|
||||
exit $RC
|
||||
fi
|
||||
# Non-strict: only fail on parse errors / errors, not style notes.
|
||||
# (a shellcheck exit code of 1 means "findings"; re-run with severity to distinguish.)
|
||||
exit 0
|
||||
Executable
+169
@@ -0,0 +1,169 @@
|
||||
#!/bin/bash
|
||||
# validate-fixes.sh - READ-ONLY validation of all applied tunings.
|
||||
# Does NOT reboot, shutdown VMs, or modify anything.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
HOSTS=(pfv-tsys1 pfv-tsys3 pfv-tsys6 pfv-tsys7 pfv-tsys9)
|
||||
|
||||
echo "==================================================================="
|
||||
echo " READ-ONLY VALIDATION — $(date)"
|
||||
echo "==================================================================="
|
||||
echo ""
|
||||
|
||||
for HOST in "${HOSTS[@]}"; do
|
||||
echo "================================================================"
|
||||
echo "[$HOST]"
|
||||
echo "================================================================"
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE"
|
||||
echo ""
|
||||
continue
|
||||
fi
|
||||
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
pass=0; fail=0
|
||||
check() {
|
||||
local label="$1" actual="$2" expected="$3"
|
||||
if [ "$actual" = "$expected" ]; then
|
||||
printf " [OK] %-30s %s\n" "$label" "$actual"
|
||||
pass=$((pass+1))
|
||||
else
|
||||
printf " [FAIL] %-30s got=%s want=%s\n" "$label" "$actual" "$expected"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
}
|
||||
|
||||
# 1. CPU governor
|
||||
gov=$(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "n/a")
|
||||
if [ "$gov" = "n/a" ]; then
|
||||
printf " [SKIP] %-30s %s\n" "CPU governor" "(no cpufreq driver — OK for server BIOS)"
|
||||
else
|
||||
check "CPU governor" "$gov" "performance"
|
||||
fi
|
||||
|
||||
# 2. vm.swappiness
|
||||
swap=$(sysctl -n vm.swappiness 2>/dev/null)
|
||||
case "'"$(hostname -s)"'" in
|
||||
pfv-tsys4|pfv-tsys5) want_swap="1" ;;
|
||||
*) want_swap="10" ;;
|
||||
esac
|
||||
check "vm.swappiness" "$swap" "$want_swap"
|
||||
|
||||
# 3. TCP congestion control
|
||||
cc=$(sysctl -n net.ipv4.tcp_congestion_control 2>/dev/null)
|
||||
check "tcp_congestion_control" "$cc" "bbr"
|
||||
|
||||
# 4. default_qdisc (paired with BBR)
|
||||
qd=$(sysctl -n net.core.default_qdisc 2>/dev/null)
|
||||
check "net.core.default_qdisc" "$qd" "fq"
|
||||
|
||||
# 5. tuned profile
|
||||
if command -v tuned-adm >/dev/null 2>&1; then
|
||||
tuned=$(tuned-adm active 2>/dev/null | awk -F: "/Current active/{gsub(/^[ \t]+/,\"\",\$2); print \$2}")
|
||||
case "'"$(hostname -s)"'" in
|
||||
pfv-tsys4|pfv-tsys5) want_tuned="throughput-performance" ;;
|
||||
*) want_tuned="virtual-host" ;;
|
||||
esac
|
||||
check "tuned-adm profile" "$tuned" "$want_tuned"
|
||||
else
|
||||
printf " [FAIL] %-30s not installed\n" "tuned-adm"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
# 6. bond0 hash policy (if bond exists)
|
||||
if [ -r /proc/net/bonding/bond0 ]; then
|
||||
hash=$(grep "Transmit Hash" /proc/net/bonding/bond0 2>/dev/null | awk "{print \$4}")
|
||||
check "bond0 xmit_hash_policy" "$hash" "layer3+4"
|
||||
ports=$(grep "Number of ports" /proc/net/bonding/bond0 2>/dev/null | awk "{print \$4}")
|
||||
printf " [INFO] %-30s %s ports active\n" "bond0 LACP ports" "$ports"
|
||||
else
|
||||
printf " [SKIP] %-30s %s\n" "bond0 hash" "(no bond0 — single NIC host)"
|
||||
fi
|
||||
|
||||
# 7. NFS mount options (nconnect + noatime)
|
||||
nfs_first=$(nfsstat -m 2>/dev/null | head -3 | tail -1)
|
||||
if echo "$nfs_first" | grep -q "nconnect=4"; then
|
||||
printf " [OK] %-30s nconnect=4 active\n" "NFS nconnect"
|
||||
pass=$((pass+1))
|
||||
elif echo "$nfs_first" | grep -q "relatime"; then
|
||||
printf " [FAIL] %-30s still relatime (needs reboot/mount)\n" "NFS nconnect"
|
||||
fail=$((fail+1))
|
||||
elif [ -z "$nfs_first" ]; then
|
||||
printf " [WARN] %-30s no NFS mounts (lazy — start a VM)\n" "NFS nconnect"
|
||||
else
|
||||
printf " [FAIL] %-30s unexpected: %s\n" "NFS nconnect" "$nfs_first"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
if echo "$nfs_first" | grep -q "noatime"; then
|
||||
printf " [OK] %-30s noatime active\n" "NFS noatime"
|
||||
pass=$((pass+1))
|
||||
elif [ -n "$nfs_first" ]; then
|
||||
printf " [FAIL] %-30s not noatime\n" "NFS noatime"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
# 8. NFS TCP connection count
|
||||
nfs_conns=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
|
||||
if [ "$nfs_conns" -ge 8 ]; then
|
||||
printf " [OK] %-30s %s connections\n" "NFS TCP conns" "$nfs_conns"
|
||||
pass=$((pass+1))
|
||||
elif [ "$nfs_conns" -gt 0 ]; then
|
||||
printf " [WARN] %-30s %s (expect 8 with nconnect=4)\n" "NFS TCP conns" "$nfs_conns"
|
||||
else
|
||||
printf " [WARN] %-30s 0 (lazy mounts — start a VM)\n" "NFS TCP conns"
|
||||
fi
|
||||
|
||||
# 9. sysctl persistence
|
||||
if [ -r /etc/sysctl.d/99-perfopt.conf ]; then
|
||||
printf " [OK] %-30s /etc/sysctl.d/99-perfopt.conf\n" "sysctl persistence"
|
||||
pass=$((pass+1))
|
||||
else
|
||||
printf " [FAIL] %-30s missing\n" "sysctl persistence"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
# 10. Observability packages
|
||||
for cmd in sar jq numactl nvme mtr bmon; do
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
printf " [FAIL] %-30s not installed\n" "obs: $cmd"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
done
|
||||
if command -v sar >/dev/null 2>&1 && command -v jq >/dev/null 2>&1 && \
|
||||
command -v numactl >/dev/null 2>&1 && command -v nvme >/dev/null 2>&1 && \
|
||||
command -v mtr >/dev/null 2>&1 && command -v bmon >/dev/null 2>&1; then
|
||||
printf " [OK] %-30s all installed\n" "observability packages"
|
||||
pass=$((pass+1))
|
||||
fi
|
||||
|
||||
# 11. Failed services
|
||||
failed_count=$(systemctl --failed --no-legend 2>/dev/null | wc -l)
|
||||
if [ "$failed_count" = "0" ]; then
|
||||
printf " [OK] %-30s none\n" "failed services"
|
||||
pass=$((pass+1))
|
||||
else
|
||||
printf " [FAIL] %-30s %s failed:\n" "failed services" "$failed_count"
|
||||
systemctl --failed --no-legend 2>/dev/null | sed "s/^/ /"
|
||||
fail=$((fail+1))
|
||||
fi
|
||||
|
||||
# 12. VM status (read-only — just report)
|
||||
running_vms=$(qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"" | wc -l)
|
||||
stopped_vms=$(qm list 2>/dev/null | awk "NR>1 && \$3!=\"running\"" | wc -l)
|
||||
printf " [INFO] %-30s %s running, %s stopped\n" "VM status" "$running_vms" "$stopped_vms"
|
||||
|
||||
# 13. uptime
|
||||
printf " [INFO] %-30s %s\n" "uptime" "$(uptime | sed "s/.*up //" | sed "s/,.*//")"
|
||||
|
||||
echo ""
|
||||
echo " RESULT: $pass passed, $fail failed"
|
||||
'
|
||||
echo ""
|
||||
done
|
||||
|
||||
echo "==================================================================="
|
||||
echo " SUMMARY"
|
||||
echo "==================================================================="
|
||||
Executable
+130
@@ -0,0 +1,130 @@
|
||||
#!/bin/bash
|
||||
###############################################################################
|
||||
# validate-vms.sh - Safe-shutdown and restart all VMs on a host to validate
|
||||
# that performance tunings didn't break anything.
|
||||
#
|
||||
# For each VM:
|
||||
# 1. qm shutdown <vmid> --timeout 120 (ACPI safe shutdown)
|
||||
# 2. Wait for stopped state
|
||||
# 3. qm start <vmid>
|
||||
# 4. Wait for running state
|
||||
# 5. Check qm agent responds (if agent enabled)
|
||||
#
|
||||
# Usage: bash validate-vms.sh <host> [host...]
|
||||
###############################################################################
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=10 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
for HOST in "$@"; do
|
||||
echo "================================================================"
|
||||
echo "[$HOST] VM SAFE-SHUTDOWN/RESTART VALIDATION"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo " UNREACHABLE — skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
# Get list of running VMs
|
||||
VM_LIST=$(ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null | awk "NR>1 && \$3==\"running\"{print \$1}"')
|
||||
if [ -z "$VM_LIST" ]; then
|
||||
echo " No running VMs — nothing to validate"
|
||||
continue
|
||||
fi
|
||||
|
||||
VM_COUNT=$(echo "$VM_LIST" | wc -w)
|
||||
echo " Found $VM_COUNT running VM(s): $(echo "$VM_LIST" | tr '\n' ' ')"
|
||||
echo ""
|
||||
|
||||
# --- Phase 1: Safe shutdown all VMs ---
|
||||
echo "--- PHASE 1: Safe shutdown all VMs (120s timeout each) ---"
|
||||
for vmid in $VM_LIST; do
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] shutting down... "
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" "qm shutdown $vmid --timeout 120 --forceStop 1" 2>&1 | head -1
|
||||
done
|
||||
|
||||
# Wait for all to stop (max 180s total)
|
||||
echo ""
|
||||
echo -n " Waiting for all VMs to stop"
|
||||
WAIT_DEADLINE=$(( $(date +%s) + 180 ))
|
||||
while [ "$(date +%s)" -lt "$WAIT_DEADLINE" ]; do
|
||||
echo -n "."
|
||||
all_stopped=1
|
||||
for vmid in $VM_LIST; do
|
||||
status=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm status $vmid 2>/dev/null | awk '{print \$2}'")
|
||||
if [ "$status" = "running" ]; then
|
||||
all_stopped=0
|
||||
break
|
||||
fi
|
||||
done
|
||||
[ "$all_stopped" = "1" ] && break
|
||||
sleep 5
|
||||
done
|
||||
echo " done"
|
||||
echo ""
|
||||
|
||||
# Show stopped state
|
||||
echo "--- VM status after shutdown ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
||||
echo ""
|
||||
|
||||
# --- Phase 2: Start all VMs ---
|
||||
echo "--- PHASE 2: Start all VMs ---"
|
||||
for vmid in $VM_LIST; do
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] starting... "
|
||||
if start_output=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm start $vmid" 2>&1); then
|
||||
echo "OK"
|
||||
else
|
||||
echo "FAILED: $start_output"
|
||||
fi
|
||||
done
|
||||
|
||||
# Wait 20s for VMs to fully start
|
||||
echo ""
|
||||
echo " Waiting 20s for VMs to boot..."
|
||||
sleep 20
|
||||
|
||||
# --- Phase 3: Verify all VMs running ---
|
||||
echo ""
|
||||
echo "--- PHASE 3: Verification ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list'
|
||||
echo ""
|
||||
|
||||
# Check NFS mounts still healthy
|
||||
echo "--- NFS mount health ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" '
|
||||
mount_count=$(nfsstat -m 2>/dev/null | grep -c "^/mnt")
|
||||
conn_count=$(ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l)
|
||||
echo " NFS mounts: $mount_count"
|
||||
echo " NFS TCP connections: $conn_count"
|
||||
if [ "$mount_count" -gt 0 ]; then
|
||||
echo " First mount options:"
|
||||
nfsstat -m 2>/dev/null | head -2 | tail -1 | sed "s/^/ /"
|
||||
fi
|
||||
'
|
||||
echo ""
|
||||
|
||||
# Check guest agent responsiveness (if agent enabled)
|
||||
echo "--- Guest agent check (VMs with agent:1) ---"
|
||||
for vmid in $VM_LIST; do
|
||||
agent=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | grep -c '^agent: 1'")
|
||||
if [ "$agent" = "1" ]; then
|
||||
name=$(ssh "${SSH_OPTS[@]}" "root@$HOST" "qm config $vmid 2>/dev/null | awk -F: '/^name:/{gsub(/^ /,\"\");print \$2}'")
|
||||
echo -n " [$vmid $name] agent ping... "
|
||||
if ssh "${SSH_OPTS[@]}" "root@$HOST" "timeout 10 qm agent $vmid ping" >/dev/null 2>&1; then
|
||||
echo "OK"
|
||||
else
|
||||
echo "no response (VM may still be booting)"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] VALIDATION COMPLETE"
|
||||
echo "================================================================"
|
||||
echo ""
|
||||
done
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
# verify-tuning.sh - verifies apply-tunings.sh results on target hosts.
|
||||
set -uo pipefail
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
for host in "$@"; do
|
||||
echo "================================================================"
|
||||
echo "[$host] verification"
|
||||
echo "================================================================"
|
||||
|
||||
echo "--- TCP congestion control ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc 2>/dev/null'
|
||||
|
||||
echo "--- vm.swappiness ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'sysctl vm.swappiness 2>/dev/null'
|
||||
|
||||
echo "--- scaling_governor (cpu0) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null'
|
||||
|
||||
echo "--- NFS mount options (first 2 mounts) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'nfsstat -m 2>/dev/null | head -30'
|
||||
|
||||
echo "--- storage.cfg: any options lines? ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -c "options" /etc/pve/storage.cfg 2>/dev/null || echo 0'
|
||||
|
||||
echo "--- storage.cfg: NFS stanzas (first 3) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'grep -A 6 "^nfs:" /etc/pve/storage.cfg 2>/dev/null | head -25'
|
||||
|
||||
echo "--- nconnect TCP connections to NFS servers ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'ss -tn state established "( dport = :nfs or sport = :nfs )" 2>/dev/null | head -20; echo "count:"; ss -tn state established "( dport = :nfs or sport = :nfs )" 2>/dev/null | tail -n +2 | wc -l'
|
||||
|
||||
echo "--- VMs still running? ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$host" 'qm list 2>/dev/null | head -15'
|
||||
|
||||
echo ""
|
||||
done
|
||||
Executable
+58
@@ -0,0 +1,58 @@
|
||||
#!/bin/bash
|
||||
# wait-for-host.sh - polls SSH until host is back, then runs verification.
|
||||
# Usage: bash wait-for-host.sh <host>
|
||||
set -uo pipefail
|
||||
HOST="$1"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o StrictHostKeyChecking=accept-new)
|
||||
|
||||
DEADLINE=$(( $(date +%s) + 600 ))
|
||||
echo "Polling $HOST for SSH return (max 10 min)..."
|
||||
while [ "$(date +%s)" -lt "$DEADLINE" ]; do
|
||||
if ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "[$(date +%H:%M:%S)] SSH is back!"
|
||||
break
|
||||
fi
|
||||
sleep 10
|
||||
echo " [$(date +%H:%M:%S)] still down..."
|
||||
done
|
||||
|
||||
if ! ssh "${SSH_OPTS[@]}" "root@$HOST" 'echo ok' >/dev/null 2>&1; then
|
||||
echo "FAILED: $HOST not back after 10 minutes"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Waiting 30s for services to settle..."
|
||||
sleep 30
|
||||
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] POST-REBOOT VERIFICATION"
|
||||
echo "================================================================"
|
||||
echo "--- Uptime (should be < 5 min) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'uptime'
|
||||
echo ""
|
||||
echo "--- TCP congestion control (expect bbr) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl net.ipv4.tcp_congestion_control net.core.default_qdisc'
|
||||
echo ""
|
||||
echo "--- vm.swappiness ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'sysctl vm.swappiness'
|
||||
echo ""
|
||||
echo "--- scaling_governor ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo "(no cpufreq driver)"'
|
||||
echo ""
|
||||
echo "--- NFS mount options (looking for nconnect=4 + noatime) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'nfsstat -m 2>/dev/null | head -16'
|
||||
echo ""
|
||||
echo "--- NFS TCP connection count to :2049 (expect ~8 = 4 per server × 2 servers) ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null'
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'ss -tn state established "( dport = :2049 )" 2>/dev/null | tail -n +2 | wc -l'
|
||||
echo ""
|
||||
echo "--- Running VMs ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'qm list 2>/dev/null'
|
||||
echo ""
|
||||
echo "--- Failed services ---"
|
||||
ssh "${SSH_OPTS[@]}" "root@$HOST" 'systemctl --failed --no-legend 2>/dev/null | head -10'
|
||||
echo ""
|
||||
echo "================================================================"
|
||||
echo "[$HOST] DONE"
|
||||
echo "================================================================"
|
||||
Reference in New Issue
Block a user